Breaking changes in instances selector

Structure has changes
Add fetch nodeinfo and avalaibility for all instances in list
Can remove failed instances

funkwhale_cli.py

@@ -8,7 +8,7 @@ from src.fw_tracks import list_tracks
 from src.fw_channels import list_channels
 from src.fw_playlists import list_playlists
 from src.fw_recents import list_fav_or_history
-import src.settings as settings
+from src.fw_instances import instances_menu
 import src.mpv_control
 import json
 import os
@@ -89,25 +89,7 @@ def main():
         if selected == 'Recently listened':
             list_fav_or_history(is_history_view=True)
         if selected == 'Switch instance':
-            with open('config.json', 'rt') as f:
+            instances_menu()
-                conf = json.loads(f.read())
-                if conf.get('automatic_fetch_new_instances'):
-                    public_server_list_instances = settings.get_new_funkwhale_servers()
-                    new_ins_count = len(settings.get_new_funkwhale_servers())
-                else:
-                    public_server_list_instances = []
-                    new_ins_count = 'Disabled'
-                list_instances = conf.get(
-                    'public_list_instances') + public_server_list_instances
-                settings.set_config('public_list_instances', list_instances)
-                instance = fzf.prompt(list_instances,
-                                      '--header='+quote(f'Select instance\nNew instances: {new_ins_count}'))
-                if instance == []:
-                    continue
-                else:
-                    instance = instance[0]
-
-                current_instance.select_instance(instance)
         if selected == 'Sign in':
             print(f'''
 If You want sign in, please visit:

src/fw_instances.py

@@ -0,0 +1,141 @@
+from src.fw_api import current_instance
+import src.settings as settings
+from pyfzf.pyfzf import FzfPrompt
+from shlex import quote
+from loguru import logger
+import json
+import time
+import concurrent
+import requests
+
+fzf = FzfPrompt()
+
+
+@logger.catch
+def get_new_funkwhale_servers():
+    # Uses official API network.funkwhale.audio for getting new instances
+    public_server_api = 'https://network.funkwhale.audio/dashboards/api/tsdb/query'
+    now = int(time.time())
+    timeback = now - 86400
+
+    request_public_servers = {
+        'from': f"{timeback}",
+        'to': f"{now}",
+        'queries': [
+            {
+                'refId': "A",
+                'intervalMs': 60000,
+                'maxDataPoints': 1174,
+                'datasourceId': 1,
+                'rawSql': "SELECT * FROM (\n  SELECT\n    DISTINCT on (c.domain) c.domain as \"Name\",\n    c.up as \"Is up\",\n    coalesce(c.open_registrations, false) as \"Open registrations\",\n    coalesce(anonymous_can_listen, false) as \"Anonymous can listen\",\n    coalesce(c.usage_users_total, 0) as \"Total users\",\n    coalesce(c.usage_users_active_month, 0) as \"Active users (this month)\",\n    coalesce(c.software_version_major, 0)::text || '.' || coalesce(c.software_version_minor, 0)::text || '.' || coalesce(c.software_version_patch, 0)::text as \"Version\",\n    c.time as \"Last checked\",\n    d.first_seen as \"First seen\"\n  FROM checks as c\n  INNER JOIN domains AS d ON d.name = c.domain\n  WHERE d.blocked = false AND c.up = true AND c.time > now() - INTERVAL '7 days'\n      AND c.anonymous_can_listen IN ('true')\n      AND c.open_registrations IN ('true','false')\n\n  ORDER BY c.domain, c.time DESC\n) as t ORDER BY \"Active users (this month)\"  DESC",
+                'format': "table"
+            }
+        ]
+    }
+    try:
+        r = requests.post(public_server_api, json=request_public_servers)
+        results = r.json()
+        new_instances = {}
+        if results:
+            new_instances_list = results['results']['A']['tables'][0]['rows']
+            for i in new_instances_list:
+                anonymousCanListen = i[1]
+                if anonymousCanListen:
+                    new_instances[i[0]] = f'{anonymousCanListen} | ?'
+
+        for i in get_new_funkwhale_servers_fediverse_observer():
+            new_instances[i] = "?"
+        return new_instances
+    except: # If any errors then return empty list
+        return {}
+
+
+def get_new_funkwhale_servers_fediverse_observer():
+    try:
+        graphQL_request = {
+                'query':
+                '{\n  nodes(softwarename: \"funkwhale\") {\n    domain\n    metanodeinfo\n  }\n}'
+                }
+        r = requests.post('https://api.fediverse.observer/',
+                          headers={'Accept-Encoding': 'gzip, deflate'},
+                          json=graphQL_request)
+        new_instances = []
+        for i in r.json()['data']['nodes']:
+            if i.get('metanodeinfo'):
+                auth_no_required = json.loads(i['metanodeinfo'])['library']['anonymousCanListen']
+                if auth_no_required and i['domain']:
+                    new_instances.append(i['domain'])
+        return new_instances
+    except:
+        return []
+
+
+def fetch_instances_nodeinfo_and_avalaibility(instances):
+    extended_instances_info = {}
+
+    def request_nodeinfo(instance):
+        return requests.get('https://' + instance + '/api/v1/instance/nodeinfo/2.0/',
+                            headers={
+                                'Accept-Encoding': 'gzip, brotli, deflate',
+                                'User-Agent': 'funkwhale-cli/latest-commit; +https://git.phreedom.club/localhost_frssoft/funkwhale-cli'},
+                            timeout=5).json()
+
+    with concurrent.futures.ThreadPoolExecutor() as executor: # optimally defined number of threads
+        res = [executor.submit(request_nodeinfo, instance) for instance in instances]
+        concurrent.futures.wait(res)
+    for idx, v in enumerate(instances):
+        try:
+            data_for_instance = res[idx].result()
+            anon = data_for_instance['metadata']['library']['anonymousCanListen']
+            tracks = data_for_instance['metadata']['library']['tracks']['total']
+            extended_instances_info[v] = f'{anon} | {tracks}'
+        except:
+            extended_instances_info[v] = 'fail'
+    return extended_instances_info
+
+
+def instances_menu(fetch_manually=False, fetch_node_info=False):
+    with open('config.json', 'rt') as f:
+        conf = json.loads(f.read())
+        if conf.get('automatic_fetch_new_instances') or fetch_manually:
+            public_server_list_instances = get_new_funkwhale_servers()
+            new_ins_count = len(public_server_list_instances)
+        else:
+            public_server_list_instances = {}
+            new_ins_count = 'Disabled'
+
+    list_instances = conf.get('public_list_instances_extended')
+    if public_server_list_instances != {}:
+        list_instances_merge = {**list_instances, **public_server_list_instances}
+        settings.set_config('public_list_instances_extended', list_instances_merge)
+        list_instances = list_instances_merge
+
+    map_in_extend_mode = ''
+    if fetch_node_info:
+        list_instances = fetch_instances_nodeinfo_and_avalaibility([instance.split('|')[0].strip() for instance in list_instances.keys()])
+        settings.set_config('public_list_instances_extended', list_instances)
+        map_in_extend_mode = '\nmap: instance | anonymousCanListen | tracks'
+    instance_menu_selector = ['Fetch new instances',
+                              'Fetch nodeinfo and avalaibility',
+                              'Remove unreachible instances']
+
+    instance = fzf.prompt(
+            instance_menu_selector +
+            [f'{instance} | {info}' for instance, info in list_instances.items()],
+            '--header='+quote(f'Select instance\nNew instances: {new_ins_count}{map_in_extend_mode}'))
+    if instance == []:
+        return
+    else:
+        instance = instance[0].split('|')[0].strip()
+    if instance == 'Fetch new instances':
+        return instances_menu(fetch_manually=True)
+    if instance == 'Fetch nodeinfo and avalaibility':
+        return instances_menu(fetch_node_info=True)
+    if instance == 'Remove unreachible instances':
+        clean_unreach = {}
+        for ins, info in list_instances.items():
+            if 'fail' not in info.split():
+                clean_unreach[ins] = info
+        settings.set_config('public_list_instances_extended', clean_unreach)
+        return instances_menu()
+    current_instance.select_instance(instance)

src/settings.py

@@ -1,6 +1,4 @@
 import json
-import requests
-import time
 from os.path import exists
 from loguru import logger
 from pyfzf.pyfzf import FzfPrompt
@@ -11,36 +9,37 @@ conf_file = 'config.json'
 
 default_conf = {
     'instance': 'fw.ponychord.rocks',
-    'public_list_instances': [
+    'public_list_instances_extended':
-        "open.audio",
+        {
-        "funkwhale.co.uk",
+            "open.audio": None,
-        "am.pirateradio.social",
+            "funkwhale.co.uk": None,
-        "audio.liberta.vip",
+            "am.pirateradio.social": None,
-        "audio.gafamfree.party",
+            "audio.liberta.vip": None,
-        "tanukitunes.com",
+            "audio.gafamfree.party": None,
-        "funkwhale.juniorjpdj.pl",
+            "tanukitunes.com": None,
-        "audio.securetown.in.ua",
+            "funkwhale.juniorjpdj.pl": None,
-        "tavia.mle.party",
+            "audio.securetown.in.ua": None,
-        "funkwhale.thurk.org",
+            "tavia.mle.party": None,
-        "buzzworkers.com",
+            "funkwhale.thurk.org": None,
-        "soundship.de",
+            "buzzworkers.com": None,
-        "funkwhale.kameha.click",
+            "soundship.de": None,
-        "music.chosto.me",
+            "funkwhale.kameha.click": None,
-        "zik.goe.land",
+            "music.chosto.me": None,
-        "music.humanoids.be",
+            "zik.goe.land": None,
-        "music.hempton.us",
+            "music.humanoids.be": None,
-        "mizik.o-k-i.net",
+            "music.hempton.us": None,
-        "klh.radiolivre.org",
+            "mizik.o-k-i.net": None,
-        "hudba.feildel.fr",
+            "klh.radiolivre.org": None,
-        "funkwhale.mita.me",
+            "hudba.feildel.fr": None,
-        "funk.deko.cloud",
+            "funkwhale.mita.me": None,
-        "audio.graz.social",
+            "funk.deko.cloud": None,
-        "funkwhale.desmu.fr",
+            "audio.graz.social": None,
-        "listen.knsm.cc",
+            "funkwhale.desmu.fr": None,
-        "funkwhale.gegeweb.eu",
+            "listen.knsm.cc": None,
-        "shitnoise.monster"
+            "funkwhale.gegeweb.eu": None,
-    ],
+            "shitnoise.monster": None,
-    'automatic_fetch_new_instances': True,
+        },
+    'automatic_fetch_new_instances': False,
     'enable_server_transcoding': False,
     'external_transcoder_http_proxy_path': "",
     'track_activity_history': False,
@@ -97,61 +96,3 @@ def set_config(key, value):
     with open(conf_file, 'wt') as f:
         read_conf[key] = value
         f.write(json.dumps(read_conf, indent=4))
-
-@logger.catch
-def get_new_funkwhale_servers():
-    # Uses official API network.funkwhale.audio for getting new instances
-    public_server_api = 'https://network.funkwhale.audio/dashboards/api/tsdb/query'
-    now = int(time.time())
-    timeback = now - 86400
-
-    request_public_servers = {
-        'from': f"{timeback}",
-        'to': f"{now}",
-        'queries': [
-            {
-                'refId': "A",
-                'intervalMs': 60000,
-                'maxDataPoints': 1174,
-                'datasourceId': 1,
-                'rawSql': "SELECT * FROM (\n  SELECT\n    DISTINCT on (c.domain) c.domain as \"Name\",\n    c.up as \"Is up\",\n    coalesce(c.open_registrations, false) as \"Open registrations\",\n    coalesce(anonymous_can_listen, false) as \"Anonymous can listen\",\n    coalesce(c.usage_users_total, 0) as \"Total users\",\n    coalesce(c.usage_users_active_month, 0) as \"Active users (this month)\",\n    coalesce(c.software_version_major, 0)::text || '.' || coalesce(c.software_version_minor, 0)::text || '.' || coalesce(c.software_version_patch, 0)::text as \"Version\",\n    c.time as \"Last checked\",\n    d.first_seen as \"First seen\"\n  FROM checks as c\n  INNER JOIN domains AS d ON d.name = c.domain\n  WHERE d.blocked = false AND c.up = true AND c.time > now() - INTERVAL '7 days'\n      AND c.anonymous_can_listen IN ('true')\n      AND c.open_registrations IN ('true','false')\n\n  ORDER BY c.domain, c.time DESC\n) as t ORDER BY \"Active users (this month)\"  DESC",
-                'format': "table"
-            }
-        ]
-    }
-    try:
-        r = requests.post(public_server_api, json=request_public_servers)
-        results = r.json()
-        new_instances = []
-        if results:
-            new_instances_list = results['results']['A']['tables'][0]['rows']
-            exists_instances = get_config('public_list_instances')
-            for i in new_instances_list:
-                if i[0] not in exists_instances and i[1]:
-                    new_instances.append(i[0])
-        new_instances.extend(get_new_funkwhale_servers_fediverse_observer())
-        new_instances = list(dict.fromkeys(new_instances))
-        return new_instances
-    except: # If any errors then return empty list
-        return []
-
-
-def get_new_funkwhale_servers_fediverse_observer():
-    try:
-        graphQL_request = {
-                'query':
-                '{\n  nodes(softwarename: \"funkwhale\") {\n    domain\n    metanodeinfo\n  }\n}'
-                }
-        r = requests.post('https://api.fediverse.observer/',
-                          headers={'Accept-Encoding': 'gzip, deflate'},
-                          json=graphQL_request)
-        new_instances = []
-        exists_instances = get_config('public_list_instances')
-        for i in r.json()['data']['nodes']:
-            if i.get('metanodeinfo'):
-                auth_no_required = json.loads(i['metanodeinfo'])['library']['anonymousCanListen']
-                if auth_no_required and i['domain'] not in exists_instances:
-                    new_instances.append(i['domain'])
-        return new_instances
-    except:
-        return []