~metalune/public-inbox

This thread contains a patchset. You're looking at the original emails, but you may wish to use the patch review UI. Review patch
1

[PATCH 1/2] Also load videos on homepage from subscribed channels.list

southerntofu
Details
Message ID
<20210729164918.1455775-1-southerntofu@thunix.net>
DKIM signature
missing
Download raw message
Patch: +70 -12
---
 main.py              | 77 +++++++++++++++++++++++++++++++++++++++-----
 templates/index.html |  5 +--
 2 files changed, 70 insertions(+), 12 deletions(-)

diff --git a/main.py b/main.py
index 1e21c0a..9680885 100644
--- a/main.py
+++ b/main.py
@@ -94,8 +94,9 @@ cached_instance_names = Cache()
cached_account_infos = Cache()
cached_video_channel_infos = Cache()

cached_subscriptions_accounts = Cache(criteria = lambda diff: diff.total_seconds() > 60)
cached_subscriptions = Cache(criteria = lambda diff: diff.total_seconds() > 60)
cached_account_videos = Cache(criteria = lambda diff: diff.total_seconds() > 1800)
cached_channel_videos = Cache(criteria = lambda diff: diff.total_seconds() > 1800)

# cache the instance names so we don't have to send a request to the domain every time someone
# loads any site 
@@ -120,19 +121,29 @@ def get_video_channel(info):
def get_video_channel_info(name):
    return cached_video_channel_infos.get(name, get_video_channel)

# Get latest remote videos from name
# Get latest remote videos from channel name
def get_latest_channel_videos(name):
    return cached_channel_videos.get(name, latest_channel_videos)

# Refresh latest remote videos from channel name
def latest_channel_videos(name):
    print("[CACHE] Refreshing channel videos for %s" % name)
    (name, domain) = name.split('@')
    return peertube.video_channel_videos(domain, name, 0)

# Get latest remote videos from account name
def get_latest_account_videos(name):
    return cached_account_videos.get(name, latest_account_videos)

# Refresh latest remote videos from name
# Refresh latest remote videos from account name
def latest_account_videos(name):
    print("[CACHE] Refreshing acount videos for %s" % name)
    print("[CACHE] Refreshing account videos for %s" % name)
    (name, domain) = name.split('@')
    return peertube.account_videos(domain, name, 0)

# Get local accounts subscriptions, as specified in accounts.list
def get_subscriptions_accounts():
    return cached_subscriptions_accounts.get("accounts", load_subscriptions_accounts)
    return cached_subscriptions.get("accounts", load_subscriptions_accounts)

# Refresh local accounts subscriptions
def load_subscriptions_accounts(_):
@@ -145,7 +156,7 @@ def load_subscriptions_accounts(_):
        subscriptions = []
    return subscriptions

# Get the latest videos from local accounts subscriptions, ordered by most recent and with ; only return `limit` number of videos
# Get the latest videos from local accounts subscriptions, ordered by most recent; only return `limit` number of videos
def get_subscriptions_accounts_videos(limit=12):
    latest  = []
    for sub in get_subscriptions_accounts():
@@ -154,14 +165,64 @@ def get_subscriptions_accounts_videos(limit=12):
    latest.sort(key = lambda vid: dateutil.isoparse(vid["createdAt"]), reverse=True)
    return latest[0:limit]

# Get local channels subscriptions, as specified in channel.list
def get_subscriptions_channels():
    return cached_subscriptions.get("channels", load_subscriptions_channels)

# Refresh local channels subscriptions
def load_subscriptions_channels(_):
    print("[CACHE] Refreshing subscriptions channels from channels.list")
    try:
        with open('channels.list', 'r') as f:
            subscriptions = f.read().splitlines()
    except Exception as e:
        print("No `channels.list` file to load for local subscriptions")
        subscriptions = []
    return subscriptions

# Get the latest videos from local channels subscriptions, ordered by most recent; only return `limit` number of videos
def get_subscriptions_channels_videos(limit=12):
    latest  = []
    for sub in get_subscriptions_channels():
        channel_latest = get_latest_channel_videos(sub)["data"]
        latest.extend(channel_latest)
    latest.sort(key = lambda vid: dateutil.isoparse(vid["createdAt"]), reverse=True)
    return latest[0:limit]

# Get the latest videos from local channels and accounts subscriptions combined, ordered by most recent; only return `limit` number of videos; NOTE: duplicates are not handled, why would you add both an account and the corresponding channel?
def get_subscriptions_videos(limit=12):
    latest = get_subscriptions_channels_videos(limit=limit)
    latest.extend(get_subscriptions_accounts_videos(limit=limit))
    # TODO: maybe refactor so we don't have to reorder twice? Or maybe the get_ functions can take a ordered=True argument? In this case here, it would be false, because we sort after
    latest.sort(key = lambda vid: dateutil.isoparse(vid["createdAt"]), reverse=True)
    return latest[0:limit]

# Get the info about local accounts subscriptions
def get_subscriptions_accounts_info():
    return map(lambda sub: get_account_info(sub), get_subscriptions_accounts())

# Get the info about local channels subscriptions
def get_subscriptions_channels_info():
    return map(lambda sub: get_video_channel_info(sub), get_subscriptions_channels())

# Get the info about local subscriptions for accounts and channels, as a tuple of lists
def get_subscriptions_info():
    list = []
    list.extend(get_subscriptions_accounts_info())
    list.extend(get_subscriptions_channels_info())
    return list

app = Quart(__name__)

@app.route("/")
async def main():
    videos = get_subscriptions_videos(limit=12)
    # Inside subscriptions variable, you may find either an account info structure, or a channel info structure. Channels may be recognized due to `ownerAccount` property.
    subscriptions = get_subscriptions_info()
    return await render_template(
        "index.html",
        videos = get_subscriptions_accounts_videos(),
        subscriptions = map(lambda sub: get_account_info(sub), get_subscriptions_accounts())
        videos=videos,
        subscriptions=subscriptions,
    )

@app.route("/search", methods = ["POST"])
diff --git a/templates/index.html b/templates/index.html
index 7ece231..e6106b8 100644
--- a/templates/index.html
+++ b/templates/index.html
@@ -17,12 +17,9 @@
            </form>{% if videos|length > 0 %}
            <hr>
            <h2>{{ videos|length }} latest videos from local subscriptions</h2>
            <p>{% for sub in subscriptions %}{% if not loop.first %}, {% endif %}<a href="/{{ sub.host }}/accounts/{{ sub.name }}">{{ sub.displayName }} (@{{ sub.name }}@{{ sub.host }})</a>{% endfor %}</p>
            <p>{% for sub in subscriptions %}{% if not loop.first %}, {% endif %}<a href="/{{ sub.host }}{% if sub.ownerAccount %}/video-channels{% else %}/accounts{% endif %}/{{ sub.name }}">{{ sub.displayName }} (@{{ sub.name }}@{{ sub.host }})</a>{% endfor %}</p>
            <hr>
            <div id="wrap">
                {#{% for vid in videos %}
                    <pre>{{ vid|pprint }}</pre>
                {% endfor %}#}
                {% for vid in videos %}
                <div class="result-wrapper">
                    <a href="/{{ vid.account.host }}/videos/watch/{{ vid.uuid }}">
-- 
2.30.2

[PATCH 2/2] Allow comments and multiple account/channel formats in subscriptions

southerntofu
Details
Message ID
<20210729164918.1455775-2-southerntofu@thunix.net>
In-Reply-To
<20210729164918.1455775-1-southerntofu@thunix.net> (view parent)
DKIM signature
missing
Download raw message
Patch: +42 -12
---
 main.py | 54 ++++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 42 insertions(+), 12 deletions(-)

diff --git a/main.py b/main.py
index 9680885..89dae6d 100644
--- a/main.py
+++ b/main.py
@@ -147,14 +147,7 @@ def get_subscriptions_accounts():

# Refresh local accounts subscriptions
def load_subscriptions_accounts(_):
    print("[CACHE] Refreshing subscriptions accounts from accounts.list")
    try:
        with open('accounts.list', 'r') as f:
            subscriptions = f.read().splitlines()
    except Exception as e:
        print("No `accounts.list` file to load for local subscriptions")
        subscriptions = []
    return subscriptions
    return load_subscriptions("accounts")

# Get the latest videos from local accounts subscriptions, ordered by most recent; only return `limit` number of videos
def get_subscriptions_accounts_videos(limit=12):
@@ -171,14 +164,51 @@ def get_subscriptions_channels():

# Refresh local channels subscriptions
def load_subscriptions_channels(_):
    print("[CACHE] Refreshing subscriptions channels from channels.list")
    return load_subscriptions("channels")

# Load subscriptions from a file called `kind`.list (60s cache)
def load_subscriptions(kind):
    print("[CACHE] Refreshing subscriptions %s from %s.list" % (kind, kind))
    try:
        with open('channels.list', 'r') as f:
            subscriptions = f.read().splitlines()
        with open(kind + '.list', 'r') as f:
            subscriptions = map(find_subscription, f.read().splitlines())
    except Exception as e:
        print("No `channels.list` file to load for local subscriptions")
        subscriptions = []
    return subscriptions
    # Remove comment entries and empty lines
    return filter(lambda entry: entry != '', subscriptions)

# Builds a unified id@server from one of those syntaxes, additionally stripping extra whitespace and ignoring `#` as comments:
#   - id@server
#   - @id@server
#   - http(s)://server/c/id
#   - http(s)://server/a/id
def find_subscription(request):
    identifier = request
    identifier = identifier.split('#')[0].strip()
    # Comment line is returned as empty string
    if identifier == '': return ''
    if identifier.startswith('@'):
        # Strip @ from identifier
        return identifier[1:]
    if identifier.startswith('http'):
        identifier = identifier[4:]
        # HTTPS?
        if identifier.startswith('s'): identifier = identifier[1:]
        # Remove ://
        identifier = identifier[3:]
        parts = identifier.split('/')
        domain = parts[0]
        if parts[1] == 'a' or parts[1] == 'c':
            # Account or channel found, take the next part
            return parts[2] + '@' + domain
    else:
        # Just check there's an @ in there and it should be fine
        if '@' in identifier:
            return identifier
    # No match was found, we don't understand this URL
    print("[WARN] Identifier not understood from local subscriptions:\n%s" % request)
    return ''

# Get the latest videos from local channels subscriptions, ordered by most recent; only return `limit` number of videos
def get_subscriptions_channels_videos(limit=12):
-- 
2.30.2
Reply to thread Export thread (mbox)