Coverage for aprender.py: 100%
189 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-10-01 05:56 +0000
1# This file is part of Vallenato.fr.
2#
3# Vallenato.fr is free software: you can redistribute it and/or modify
4# it under the terms of the GNU Affero General Public License as published by
5# the Free Software Foundation, either version 3 of the License, or
6# (at your option) any later version.
7#
8# Vallenato.fr is distributed in the hope that it will be useful,
9# but WITHOUT ANY WARRANTY; without even the implied warranty of
10# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11# GNU Affero General Public License for more details.
12#
13# You should have received a copy of the GNU Affero General Public License
14# along with Vallenato.fr. If not, see <http://www.gnu.org/licenses/>.
16import json
17import logging
18import os
19import re
20import readline
21import shutil
22import sys
23import webbrowser
24from urllib.error import HTTPError
26from pytube import YouTube
27from slugify import slugify
29logger = logging.getLogger(__name__)
31# File that contains the list of available tutorials
32TUTORIALES_DATA_FILE = "../website/src/aprender/tutoriales.js"
35def get_tutorial_info():
36 """Retrieve the information of the new tutorial"""
37 # What is the YouTube tutorial video?
38 (tutorial_id, tutorial_url) = get_youtube_url("tutorial")
39 # What is the YouTube full video?
40 (full_video_id, full_video_url) = get_youtube_url("full")
41 # Song title, author name and the tutorial creator's name and YouTube channel
42 (song_title, song_author, tutocreator, tutocreator_channel, yt_tutorial_video) = (
43 get_title_author_tutocreator_and_channel(tutorial_url)
44 )
45 # Tutorial's slug
46 tutorial_slug = get_tutorial_slug(song_title)
47 return (
48 tutorial_id,
49 tutorial_url,
50 full_video_id,
51 full_video_url,
52 song_title,
53 song_author,
54 tutocreator,
55 tutocreator_channel,
56 yt_tutorial_video,
57 tutorial_slug,
58 )
61def get_youtube_url(type):
62 """Extract video ID and Normalize URL"""
63 video_id = None
64 video_url = None
65 s = input(f"Enter the ID or URL of the {type} video ('q' to quit): ")
66 while not video_id:
67 if s.lower() == "q":
68 print("Exiting...")
69 sys.exit(10)
70 video_id = youtube_url_validation(s)
71 if not video_id:
72 s = input(f"Invalid {type} video URL, please try again ('q' to quit): ")
73 video_url = f"https://www.youtube.com/watch?v={video_id}"
74 return (video_id, video_url)
77def youtube_url_validation(url):
78 """Check that it is a valid YouTube URL.
79 Inspired from https://stackoverflow.com/a/19161373
80 """
81 # Accept just the YouTube ID
82 if re.match("^[a-zA-Z0-9_-]{11}$", url):
83 return url
84 youtube_regex = (
85 r"(https?://)?(www\.)?"
86 r"(youtube|youtu|youtube-nocookie)\.(com|be)/"
87 r"(watch\?v=|embed/|v/|.+\?v=)?([a-zA-Z0-9_-]{11})"
88 )
89 youtube_regex_match = re.match(youtube_regex, url)
90 if youtube_regex_match:
91 return youtube_regex_match.group(6)
92 return youtube_regex_match
95def get_title_author_tutocreator_and_channel(url):
96 logger.debug(f"Downloading information from tutorial video '{url}'.")
97 yt = YouTube(url)
99 # Extract the title
100 song_title = rlinput("Song title ('q' to quit): ", yt.title)
101 if song_title.lower() == "q":
102 print("Exiting...")
103 sys.exit(11)
105 # Extract the author's name
106 song_author = rlinput("Song author ('q' to quit): ", yt.title)
107 if song_author.lower() == "q":
108 print("Exiting...")
109 sys.exit(12)
111 # The name of the creator of the tutorial
112 tutocreator = yt.author
114 # The YouTube channel of the creator of the tutorial
115 # TODO: this broke when migrating to pytube3
116 tutocreator_channel = "UPDATE MANUALLY"
117 # tutocreator_channel = yt.player_config_args["player_response"]["videoDetails"]["channelId"]
119 return (song_title, song_author, tutocreator, tutocreator_channel, yt)
122def rlinput(prompt, prefill=""):
123 """Provide an editable input string
124 Inspired from https://stackoverflow.com/a/36607077
125 """
126 readline.set_startup_hook(lambda: readline.insert_text(prefill))
127 try:
128 return input(prompt)
129 finally:
130 readline.set_startup_hook()
133def get_existing_tutorial_slug():
134 # Get the list of existing tutorial slugs
135 with open(TUTORIALES_DATA_FILE) as in_file:
136 # Remove the JS bits to keep only the JSON content
137 tutoriales_json_content = in_file.read()[17:-2]
138 tutoriales = json.loads(tutoriales_json_content)
139 tutorials_slugs = [t["slug"] for t in tutoriales]
140 return tutorials_slugs
143def get_tutorial_slug(song_title):
144 tutorials_slugs = get_existing_tutorial_slug()
145 tutorial_slug = get_suggested_tutorial_slug(song_title, tutorials_slugs)
146 # Propose the slug to the user
147 tutorial_slug = rlinput("Tutorial slug/nice URL ('q' to quit): ", tutorial_slug)
148 if tutorial_slug.lower() == "q":
149 print("Exiting...")
150 sys.exit(13)
151 while tutorial_slug in tutorials_slugs:
152 logger.debug(f"The slug '{tutorial_slug}' is already used.")
153 tutorial_slug = rlinput("Tutorial slug/nice URL ('q' to quit): ", tutorial_slug)
154 if tutorial_slug.lower() == "q":
155 print("Exiting...")
156 sys.exit(14)
157 return tutorial_slug
160def get_suggested_tutorial_slug(song_title, tutorials_slugs):
161 # This tutorial's default slug
162 tutorial_slug_base = slugify(song_title)
163 tutorial_slug = tutorial_slug_base
165 i = 1
166 # Even if this slug is not used, check if maybe the -1 version is already used
167 # This would be the case if there's already a -1 and -2 version
168 if (
169 tutorial_slug not in tutorials_slugs
170 and f"{tutorial_slug_base}-{i}" in tutorials_slugs
171 ):
172 tutorial_slug = f"{tutorial_slug_base}-{i}"
173 while tutorial_slug in tutorials_slugs:
174 logger.debug(f"The slug '{tutorial_slug}' is already used.")
175 i += 1
176 tutorial_slug = f"{tutorial_slug_base}-{i}"
177 return tutorial_slug
180def determine_output_folder(temp_folder, tutorial_slug):
181 output_folder = "../website/src/aprender/"
182 if temp_folder:
183 # Create a new temporary folder for this new tutorial
184 output_folder += f"temp/{tutorial_slug}/"
185 logger.debug(
186 f"This new tutorial will be created in '{output_folder}' due to --temp-folder parameter."
187 )
188 if os.path.exists(output_folder):
189 # Ask if the existing temporary folder should be deleted or the script ended
190 logger.debug("The temporary folder already exists")
191 s = input(
192 "The temporary folder already exists, enter 'Y' to delete the folder or 'N' to stop the program: "
193 )
194 valid_entry = False
195 while not valid_entry:
196 if s.lower() == "y":
197 shutil.rmtree(output_folder)
198 valid_entry = True
199 elif s.lower() == "n":
200 print("Exiting...")
201 sys.exit(15)
202 else:
203 s = input(
204 "Enter 'Y' to delete the folder or 'N' to stop the program: "
205 )
207 if not os.path.exists(output_folder):
208 os.makedirs(output_folder)
209 return output_folder
212def download_videos(
213 yt_tutorial_video, tutorial_id, full_video_id, videos_output_folder
214):
215 # Tutorial video
216 logger.info(f"Will now download the tutorial video {tutorial_id}...")
217 download_youtube_video(yt_tutorial_video, tutorial_id, videos_output_folder)
218 # Full video
219 logger.info(f"Will now download the full video {full_video_id}...")
220 # For the tutorial video we already had a Youtube object, not yet for the full video
221 video_url = f"https://www.youtube.com/watch?v={full_video_id}"
222 yt_full_video = YouTube(video_url)
223 download_youtube_video(yt_full_video, full_video_id, videos_output_folder)
226def download_youtube_video(yt, video_id, videos_output_folder):
227 # Download stream with itag 18 by default:
228 # <Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2">
229 stream = yt.streams.get_by_itag(18)
230 if not stream:
231 logger.debug("No stream available with itag 18")
232 stream = yt.streams.filter(
233 res="360p", progressive=True, file_extension="mp4"
234 ).first()
235 logger.debug(f"Stream that will be downloaded: {stream}")
236 logger.debug(f"Download folder: {videos_output_folder}")
237 try:
238 download_stream(stream, videos_output_folder, video_id)
239 except HTTPError as e:
240 logger.error(f"An HTTP error {e.code} occurred with reason: {e.reason}")
241 # Propose to download that video manually from the browser
242 webbrowser.open(f"https://y2mate.com/youtube/{video_id}", new=2, autoraise=True)
243 return False
244 return True
247def download_stream(stream, videos_output_folder, video_id):
248 stream.download(videos_output_folder, video_id)
251def generate_new_tutorial_info(
252 tutorial_slug, song_author, song_title, tutorial_id, full_video_id
253):
254 new_tutorial_info = f"""{{
255 "slug": "{tutorial_slug}",
256 "author": "{song_author}",
257 "title": "{song_title}",
258 "videos": [
259 {{"id": "{tutorial_id}", "start": 0, "end": 999}}
260 ],
261 "videos_full_tutorial": [],
262 "full_version": "{full_video_id}"
263 }}"""
264 return new_tutorial_info
267def update_tutoriales_data_file(tutoriales_data_file, new_tutorial_info):
268 # Read in the tutoriales data file
269 with open(tutoriales_data_file, "r") as file:
270 filedata = file.read()
271 # Add the new tutorial info to the list of tutorials
272 filedata = filedata.replace("}\n];", f"}},\n {new_tutorial_info}\n];")
273 # Save edited file
274 with open(tutoriales_data_file, "w") as file:
275 file.write(filedata)
278def index_new_tutorial_link(tutorial_slug, song_title, song_author):
279 return f"""\n <div class="card mb-3" style="max-width: 17rem;">
280 <div class="card-body">
281 <h5 class="card-title">{song_title} - {song_author}</h5>
282 <a href="{tutorial_slug}" class="stretched-link text-hide">Ver el tutorial</a>
283 </div>
284 <div class="card-footer"><small class="text-muted">NNmNNs en NN partes</small></div>
285 </div>"""
288def index_new_youtube_links(
289 song_title, song_author, tutorial_url, tutocreator_channel, tutocreator
290):
291 return f'\n <li>{song_title} - {song_author}: <a href="{tutorial_url}">Tutorial en YouTube</a> por <a href="https://www.youtube.com/channel/{tutocreator_channel}">{tutocreator}</a></li>'
294def dummy_index_update(
295 tutorial_slug,
296 song_title,
297 song_author,
298 tutorial_url,
299 tutocreator_channel,
300 tutocreator,
301 output_folder,
302):
303 dummy_index_page = f"{output_folder}index-dummy.html"
304 logger.info(
305 f"Creating a new dummy index page '{dummy_index_page}' with links to be included later in the main index page."
306 )
307 filedata = index_new_tutorial_link(tutorial_slug, song_title, song_author)
308 filedata += index_new_youtube_links(
309 song_title, song_author, tutorial_url, tutocreator_channel, tutocreator
310 )
311 with open(dummy_index_page, "w") as file:
312 file.write(filedata)
315def dummy_symlink_files(output_folder):
316 logger.debug(
317 f"Creating symlinks for the .js and .css files in the dummy folder '{output_folder}'."
318 )
319 os.symlink("../../vallenato.fr.js", f"{output_folder}vallenato.fr.js")
320 os.symlink("../../vallenato.fr.css", f"{output_folder}vallenato.fr.css")
323def update_index_page(
324 tutorial_slug,
325 song_title,
326 song_author,
327 tutorial_url,
328 tutocreator_channel,
329 tutocreator,
330):
331 logger.info("Updating the index page with links to the new tutorial page.")
332 # Read in the index page
333 with open("../website/src/aprender/index.html", "r") as file:
334 filedata = file.read()
336 # Add a link to the new tutorial's page
337 end_section = """
338 </div>
339 </div>
340 </div>
341 <div class="row">
342 <div class="col-md">
343 <h2>Otros recursos</h2>"""
344 new_link = index_new_tutorial_link(tutorial_slug, song_title, song_author)
345 tut_number = filedata.count("<!-- Tutorial ") + 1
346 # TODO: add "wrap every N on ZZ" depending on the tutorial's number
347 filedata = filedata.replace(
348 end_section,
349 f"\n <!-- Tutorial {tut_number} -->{new_link}\n{end_section}",
350 )
352 # Add links to the tutorial and the author's YouTube channel
353 end_section = "\n </ul>\n </div>\n </div>\n </div>\n </main>\n <!-- End page content -->"
354 new_link = index_new_youtube_links(
355 song_title, song_author, tutorial_url, tutocreator_channel, tutocreator
356 )
357 filedata = filedata.replace(end_section, f"{new_link}{end_section}")
359 # Save edited file
360 with open("../website/src/aprender/index.html", "w") as file:
361 file.write(filedata)
364def aprender(args):
365 # Get the information about this new tutorial
366 (
367 tutorial_id,
368 tutorial_url,
369 full_video_id,
370 _full_video_url,
371 song_title,
372 song_author,
373 tutocreator,
374 tutocreator_channel,
375 yt_tutorial_video,
376 tutorial_slug,
377 ) = get_tutorial_info()
379 # Determine the output folder (depends on the --temp-folder parameter)
380 output_folder = determine_output_folder(args.temp_folder, tutorial_slug)
382 # Get the info that will be added for the new tutorial
383 new_tutorial_info = generate_new_tutorial_info(
384 tutorial_slug, song_author, song_title, tutorial_id, full_video_id
385 )
387 if args.temp_folder:
388 # When creating the new tutorial in a temporary folder for later edition, do not update the index page
389 dummy_index_update(
390 tutorial_slug,
391 song_title,
392 song_author,
393 tutorial_url,
394 tutocreator_channel,
395 tutocreator,
396 output_folder,
397 )
398 # Symlink files so that the new template can be used from the temp folder
399 dummy_symlink_files(output_folder)
400 else:
401 # Update the index page with the links to the new tutorial and to the tuto's author page
402 update_index_page(
403 tutorial_slug,
404 song_title,
405 song_author,
406 tutorial_url,
407 tutocreator_channel,
408 tutocreator,
409 )
410 # Add the new tutorial to the list of tutorials
411 update_tutoriales_data_file(TUTORIALES_DATA_FILE, new_tutorial_info)
413 # Download the videos (both the tutorial and the full video)
414 if args.no_download:
415 logger.info(
416 "Not downloading the videos from YouTube due to --no-download parameter."
417 )
418 else:
419 videos_output_folder = f"{output_folder}videos/{tutorial_slug}/"
420 if not os.path.exists(videos_output_folder):
421 logger.debug(f"Creating folder '{videos_output_folder}'.")
422 os.makedirs(videos_output_folder)
423 download_videos(
424 yt_tutorial_video, tutorial_id, full_video_id, videos_output_folder
425 )
427 # Open the new tutorial page in the webbrowser (new tab) for edition
428 new_tutorial_page = f"http://localhost:8000/aprender/?new_tutorial={tutorial_slug}"
429 logger.debug(f"Opening new tab in web browser to '{new_tutorial_page}'")
430 webbrowser.open(new_tutorial_page, new=2, autoraise=True)