catman/yt-mls
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rearrangements, new route and minor changes

Browse Source
This commit is contained in:
Maximilian Wagner
2023-08-02 19:07:33 +02:00
parent ddf249cfe0
commit 582332b8f7
6 changed files with 159 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
backend.py
View File

@@ -1,19 +1,27 @@
import threading
from flask import g
import yt_dlp as ydl
from yt_dlp import DownloadError
import os
import zipfile
import sqlite3
from base64 import b64encode
from base64 import b64encode
from threading import Thread
from db_tools import *
from file_cache import *
# adds thread to queue and starts it
def enqueue_download(url):
# download and processing is happening in background / another thread
t = Thread(target=process_download, args=(url,))
thread_queue.append(t)
t.start()
# this is the 'controller' for the download process
def process_download(url):
# wait for previous thread if not first in list
# wait for previous thread to finish if not first / only in list
current_thread = threading.current_thread()
if len(thread_queue) > 0 and thread_queue[0] is not current_thread:
threading.Thread.join(thread_queue[thread_queue.index(current_thread) - 1])
@@ -35,9 +43,8 @@ def process_download(url):
try:
# this throws KeyError when downloading single file
for video in query['entries']:
if check_already_exists(video['id']): # todo: this shit aint tested
query_db_threaded('INSERT INTO collection(playlist, video) VALUES (:folder, :id)',
{'folder': parent + '\\', 'id': video['id']})
if check_already_exists(video['id']):
add_to_collection_if_not_added(parent, video['id'])
continue
# this throws DownloadError when not downloading playlist
@@ -50,6 +57,7 @@ def process_download(url):
# start download
download_all(parent)
thread_queue.remove(current_thread)
return
# when downloading: channel: DownloadError, single file: KeyError
@@ -63,19 +71,19 @@ def process_download(url):
# for every video in their respective tabs
for video in tab['entries']:
if check_already_exists(video['id']):
query_db_threaded('INSERT INTO collection(playlist, video) VALUES (:folder, :id)',
{'folder': parent + '\\', 'id': video['id']})
add_to_collection_if_not_added(parent, video['id'])
continue
# todo: there have been cases of duplicate urls or some with '/watch?v=@channel_name'
# but no consistency has been observed
# still works though so will not be checked for now
# there have been cases of duplicate urls or some with '/watch?v=@channel_name'
# but no consistency has been observed
# still works though so will not be checked for now
ids.append(video['id'])
titles.append(video['title'])
urls.append('https://www.youtube.com/watch?v=' + video['id'])
# start download
download_all(parent)
thread_queue.remove(current_thread)
return
# when downloading single file: KeyError
@@ -92,7 +100,6 @@ def process_download(url):
# start download
download_all()
return
# this is broad on purpose; there has been no exception thrown here _yet_
except Exception as e:
@@ -100,7 +107,6 @@ def process_download(url):
# todo: a site with (not) finished downloads (url/datetime) would be nice so you know when it's done
# downloading large playlists does take quite a while after all
# adding that entry to the site would be done -here- i guess
thread_queue.remove(current_thread)
return
@@ -108,70 +114,13 @@ def process_download(url):
# checks whether a video is already in db
def check_already_exists(video_id) -> bool:
res = query_db_threaded('SELECT name FROM video WHERE id = :id', {'id': video_id})
res = query_db_threaded('SELECT name FROM video WHERE id = :id',
{'id': video_id})
if len(res) > 0:
return True
return False
# fetches db from app context
def get_db():
db = getattr(g, '_database', None)
if db is None:
db = g._database = sqlite3.connect('files.sqlite')
db.row_factory = sqlite3.Row
return db
# used when accessing db from app context; keeps connection alive since it's used more frequently
def query_db(query, args=(), one=False):
db = get_db()
cur = db.execute(query, args)
res = cur.fetchall()
db.commit()
cur.close()
return (res[0] if res else None) if one else res
# used when accessing db from thread, since app context is thread local; does not keep connection alive
def query_db_threaded(query, args=(), one=False):
db = sqlite3.connect('files.sqlite')
cur = db.execute(query, args)
res = cur.fetchall()
db.commit()
cur.close()
db.close()
return (res[0] if res else None) if one else res
# add entries do db
def db_add(ext, parent_rowid=None, parent=None):
# if no parent was specified
if parent is None:
# insert video into db
query_db_threaded('INSERT INTO video(id, name, ext, path) VALUES (:id, :name, :ext, :path)',
{'id': ids[0], 'name': titles[0], 'ext': '.' + ext, 'path': '\\'})
# if a parent was specified
else:
# set relative path
relative_path = parent + '\\'
# if a rowid was specified
if parent_rowid is not None:
# adjust the relative path
relative_path += str(parent_rowid) + '\\'
# insert all new files into db
for i in range(len(titles)):
query_db_threaded('INSERT INTO video(id, name, ext, path) VALUES (:id, :name, :ext, :path)',
{'id': ids[i], 'name': titles[i], 'ext': '.' + ext, 'path': relative_path})
query_db_threaded('INSERT INTO collection(playlist, video) VALUES (:folder, :id)',
{'folder': relative_path, 'id': ids[i]})
return
def download_all(parent=None, ext='mp3'):
# if no new files to download, there's nothing to do here
if not len(urls) > 0: return
@@ -245,9 +194,20 @@ def download_all(parent=None, ext='mp3'):
else:
location = downloads_path()
# start actual file download
yt_download(location, ext)
# add downloaded files to db
db_add(ext, rowid_new, parent)
return
# actually downloads files
def yt_download(location, ext='mp3'):
# base download options for audio
opts = {
'quiet': False,
'quiet': True,
'windowsfilenames': True,
'outtmpl': location + '%(title)s.%(ext)s',
'format': 'bestaudio/best',
@@ -269,9 +229,6 @@ def download_all(parent=None, ext='mp3'):
except DownloadError:
pass
# add downloaded files to db
db_add(ext, rowid_new, parent)
return

70
db_tools.py Normal file
View File

@@ -0,0 +1,70 @@
from flask import g
import sqlite3
from file_cache import ids, titles
# fetches db from app context
def get_db():
db = getattr(g, '_database', None)
if db is None:
db = g._database = sqlite3.connect('files.sqlite')
db.row_factory = sqlite3.Row
return db
# used when accessing db from app context; keeps connection alive since it's used more frequently
def query_db(query, args=(), one=False):
db = get_db()
cur = db.execute(query, args)
res = cur.fetchall()
db.commit()
cur.close()
return (res[0] if res else None) if one else res
# used when accessing db from thread, since app context is thread local; does not keep connection alive
def query_db_threaded(query, args=(), one=False):
db = sqlite3.connect('files.sqlite')
cur = db.execute(query, args)
res = cur.fetchall()
db.commit()
cur.close()
db.close()
return (res[0] if res else None) if one else res
# add entries do db
def db_add(ext, parent_rowid=None, parent=None):
# if no parent was specified
if parent is None:
# insert video into db
query_db_threaded('INSERT INTO video(id, name, ext, path) VALUES (:id, :name, :ext, :path)',
{'id': ids[0], 'name': titles[0], 'ext': '.' + ext, 'path': '\\'})
# if a parent was specified
else:
# set relative path
relative_path = parent + '\\'
# if a rowid was specified
if parent_rowid is not None:
# adjust the relative path
relative_path += str(parent_rowid) + '\\'
# insert all new files into db
for i in range(len(titles)):
query_db_threaded('INSERT INTO video(id, name, ext, path) VALUES (:id, :name, :ext, :path)',
{'id': ids[i], 'name': titles[i], 'ext': '.' + ext, 'path': relative_path})
query_db_threaded('INSERT INTO collection(playlist, video) VALUES (:folder, :id)',
{'folder': relative_path, 'id': ids[i]})
return
def add_to_collection_if_not_added(parent, video_id):
exists = query_db_threaded('SELECT * FROM collection WHERE playlist = :folder AND video = :id',
{'folder': parent + '\\', 'id': video_id})
if not len(exists) > 0:
query_db_threaded('INSERT INTO collection(playlist, video) VALUES (:folder, :id)',
{'folder': parent + '\\', 'id': video_id})

31
frontend.py
View File

@@ -1,13 +1,12 @@
from __future__ import unicode_literals
from threading import Thread
from flask import Blueprint, request, render_template, flash, send_from_directory, current_app
from flask_nav3 import Nav
from flask_nav3.elements import Navbar, View
from forms.download import DownloadForm
from backend import query_db, process_download, zip_folder, downloads_path
from backend import zip_folder, downloads_path, enqueue_download
from db_tools import query_db
from file_cache import *
frontend = Blueprint('frontend', __name__)
@@ -50,14 +49,12 @@ def downloader():
return render_template('downloader.html', form=form, ytLink=valid_link, titles=titles, urls=urls,
amount=len(titles))
# download and processing is happening in background / another thread
t = Thread(target=process_download, args=(url,))
thread_queue.append(t)
t.start()
# kick off download process
enqueue_download(url)
# show download start confirmation
flash('Download started and will continue in background.')
return render_template('new-downloads.html', titles=titles, urls=urls, amount=len(titles))
flash('Download enqueued and will finish in background.')
return render_template('feedback-simple.html', titles=titles, urls=urls, amount=len(titles))
# downloads a single file
@@ -81,7 +78,21 @@ def download(file_path):
@frontend.route('/update', methods=['GET', 'POST'])
def updater():
return render_template('updater.html')
downloads = query_db('SELECT name, url FROM updatelist INNER JOIN playlist ON updatelist.ROWID = playlist.ROWID')
return render_template('updater.html', downloads=downloads)
@frontend.route('/update/<int:url_rowid>')
def update(url_rowid):
url = query_db('SELECT url FROM updatelist WHERE ROWID = :url_rowid',
{'url_rowid': url_rowid})[0][0]
# kick off download process
enqueue_download(url)
# show download start confirmation
flash('Update enqueued and will finish in background.')
return render_template('feedback-simple.html', titles=titles, urls=urls, amount=len(titles))
@frontend.route('/library', methods=['GET'])

6
schema.sql
View File

@@ -42,3 +42,9 @@ CREATE TABLE IF NOT EXISTS collection (
video TEXT NOT NULL
);
/*
- user-input url, only playlists get added
*/
CREATE TABLE IF NOT EXISTS updatelist (
url TEXT PRIMARY KEY NOT NULL
);

0
templates/new-downloads.html → templates/feedback-simple.html
View File

35
templates/updater.html
View File

@@ -3,14 +3,33 @@
{% block content %}
{{ super() }}
<div class="container">
{%- with messages = get_flashed_messages(with_categories=True) %}
{%- if messages %}
<div class="row">
<div class="col-md-12">
{{utils.flashed_messages(messages)}}
{% if downloads %}
<div class="card" style="width: 100%">
<ul class="list-group list-group-flush">
<li class="list-group-item">
<div class="row">
<div class="col-md-5">
<table id="videos" class="table">
<thead>
<tr>
<th scope="col" class="text-center">Title</th>
<th scope="col" class="text-center">Download</th>
</tr>
</thead>
<tbody>
{% for d in downloads %}
<tr>
<td class="text-center">{{ d[0] }}</td>
<td class="text-center"><a href="/download/{{ video['path'] + video['name'] + video['ext'] }}" download>Link</a></td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
{%- endif %}
{%- endwith %}
</li>
</ul>
</div>
{% endif %}
</div>
{%- endblock %}