1#!/usr/bin/env python 

2# -*- coding: utf-8 -*- 

3 

4# This file is part of Vallenato.fr. 

5# 

6# Vallenato.fr is free software: you can redistribute it and/or modify 

7# it under the terms of the GNU Affero General Public License as published by 

8# the Free Software Foundation, either version 3 of the License, or 

9# (at your option) any later version. 

10# 

11# Vallenato.fr is distributed in the hope that it will be useful, 

12# but WITHOUT ANY WARRANTY; without even the implied warranty of 

13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

14# GNU Affero General Public License for more details. 

15# 

16# You should have received a copy of the GNU Affero General Public License 

17# along with Vallenato.fr. If not, see <http://www.gnu.org/licenses/>. 

18 

19import sys 

20import re 

21import readline 

22from slugify import slugify 

23import os 

24import shutil 

25from pytube import YouTube 

26import logging 

27import webbrowser 

28from urllib.error import HTTPError 

29import json 

30 

31# File that contains the list of available tutorials 

32TUTORIALES_DATA_FILE = "../website/src/aprender/tutoriales.js" 

33 

34def get_tutorial_info(): 

35 """Retrieve the information of the new tutorial""" 

36 # What is the YouTube tutorial video? 

37 (tutorial_id, tutorial_url) = get_youtube_url("tutorial") 

38 # What is the YouTube full video? 

39 (full_video_id, full_video_url) = get_youtube_url("full") 

40 # Song title, author name and the tutorial creator's name and YouTube channel 

41 (song_title, song_author, tutocreator, tutocreator_channel, yt_tutorial_video) = get_title_author_tutocreator_and_channel(tutorial_url) 

42 # Tutorial's slug 

43 tutorial_slug = get_tutorial_slug(song_title) 

44 return (tutorial_id, tutorial_url, full_video_id, full_video_url, song_title, song_author, tutocreator, tutocreator_channel, yt_tutorial_video, tutorial_slug) 

45 

46def get_youtube_url(type): 

47 """Extract video ID and Normalize URL""" 

48 video_id = None 

49 video_url = None 

50 s = input("Enter the ID or URL of the %s video ('q' to quit): " % type) 

51 while not video_id: 

52 if s.lower() == "q": 

53 print("Exiting...") 

54 sys.exit(10) 

55 video_id = youtube_url_validation(s) 

56 if not video_id: 

57 s = input("Invalid %s video URL, please try again ('q' to quit): " % type) 

58 video_url = "https://www.youtube.com/watch?v=%s" % video_id 

59 return (video_id, video_url) 

60 

61def youtube_url_validation(url): 

62 """Check that it is a valid YouTube URL. 

63 Inspired from https://stackoverflow.com/a/19161373 

64 """ 

65 # Accept just the YouTube ID 

66 if re.match("^[a-zA-Z0-9_-]{11}$", url): 

67 return url 

68 youtube_regex = ( 

69 r'(https?://)?(www\.)?' 

70 '(youtube|youtu|youtube-nocookie)\.(com|be)/' 

71 '(watch\?v=|embed/|v/|.+\?v=)?([a-zA-Z0-9_-]{11})') 

72 youtube_regex_match = re.match(youtube_regex, url) 

73 if youtube_regex_match: 

74 return youtube_regex_match.group(6) 

75 return youtube_regex_match 

76 

77def get_title_author_tutocreator_and_channel(url): 

78 logging.debug("Downloading information from tutorial video '%s'." % url) 

79 yt = YouTube(url) 

80 

81 # Extract the title 

82 song_title = rlinput("Song title ('q' to quit): ", yt.title) 

83 if song_title.lower() == "q": 

84 print("Exiting...") 

85 sys.exit(11) 

86 

87 # Extract the author's name 

88 song_author = rlinput("Song author ('q' to quit): ", yt.title) 

89 if song_author.lower() == "q": 

90 print("Exiting...") 

91 sys.exit(12) 

92 

93 # The name of the creator of the tutorial 

94 tutocreator = yt.author 

95 

96 # The YouTube channel of the creator of the tutorial 

97 #TODO: this broke when migrating to pytube3 

98 tutocreator_channel = "UPDATE MANUALLY" 

99 # tutocreator_channel = yt.player_config_args["player_response"]["videoDetails"]["channelId"] 

100 

101 return (song_title, song_author, tutocreator, tutocreator_channel, yt) 

102 

103def rlinput(prompt, prefill=''): 

104 """Provide an editable input string 

105 Inspired from https://stackoverflow.com/a/36607077 

106 """ 

107 readline.set_startup_hook(lambda: readline.insert_text(prefill)) 

108 try: 

109 return input(prompt) 

110 finally: 

111 readline.set_startup_hook() 

112 

113def get_existing_tutorial_slug(): 

114 # Get the list of existing tutorial slugs 

115 with open(TUTORIALES_DATA_FILE) as in_file: 

116 # Remove the JS bits to keep only the JSON content 

117 tutoriales_json_content = (in_file.read()[17:-2]) 

118 tutoriales = json.loads(tutoriales_json_content) 

119 tutorials_slugs = [t["slug"] for t in tutoriales] 

120 return tutorials_slugs 

121 

122def get_tutorial_slug(song_title): 

123 tutorials_slugs = get_existing_tutorial_slug() 

124 tutorial_slug = get_suggested_tutorial_slug(song_title, tutorials_slugs) 

125 # Propose the slug to the user 

126 tutorial_slug = rlinput("Tutorial slug/nice URL ('q' to quit): ", tutorial_slug) 

127 if tutorial_slug.lower() == "q": 

128 print("Exiting...") 

129 sys.exit(13) 

130 while tutorial_slug in tutorials_slugs: 

131 logging.debug("The slug '%s' is already used." % tutorial_slug) 

132 tutorial_slug = rlinput("Tutorial slug/nice URL ('q' to quit): ", tutorial_slug) 

133 if tutorial_slug.lower() == "q": 

134 print("Exiting...") 

135 sys.exit(14) 

136 return tutorial_slug 

137 

138def get_suggested_tutorial_slug(song_title, tutorials_slugs): 

139 # This tutorial's default slug 

140 tutorial_slug_base = slugify(song_title) 

141 tutorial_slug = tutorial_slug_base 

142 

143 i = 1 

144 if tutorial_slug not in tutorials_slugs: 

145 # Even if this slug is not used, check if maybe the -1 version is already used 

146 # This would be the case if there's already a -1 and -2 version 

147 if "%s-%d" % (tutorial_slug_base, i) in tutorials_slugs: 

148 tutorial_slug = "%s-%d" % (tutorial_slug_base, i) 

149 while tutorial_slug in tutorials_slugs: 

150 logging.debug("The slug '%s' is already used." % tutorial_slug) 

151 i += 1 

152 tutorial_slug = "%s-%d" % (tutorial_slug_base, i) 

153 return tutorial_slug 

154 

155def determine_output_folder(temp_folder, tutorial_slug): 

156 output_folder = "../website/src/aprender/" 

157 if temp_folder: 

158 # Create a new temporary folder for this new tutorial 

159 output_folder += "temp/%s/" % tutorial_slug 

160 logging.debug("This new tutorial will be created in '%s' due to --temp-folder parameter." % output_folder) 

161 if os.path.exists(output_folder): 

162 # Ask if the existing temporary folder should be deleted or the script ended 

163 logging.debug("The temporary folder already exists") 

164 s = input("The temporary folder already exists, enter 'Y' to delete the folder or 'N' to stop the program: ") 

165 valid_entry = False 

166 while not valid_entry: 

167 if s.lower() == "y": 

168 shutil.rmtree(output_folder) 

169 valid_entry = True 

170 elif s.lower() == "n": 

171 print("Exiting...") 

172 sys.exit(15) 

173 else: 

174 s = input("Enter 'Y' to delete the folder or 'N' to stop the program: ") 

175 

176 if not os.path.exists(output_folder): 

177 os.makedirs(output_folder) 

178 return output_folder 

179 

180def download_videos(yt_tutorial_video, tutorial_id, full_video_id, videos_output_folder): 

181 # Tutorial video 

182 logging.info("Will now download the tutorial video %s..." % tutorial_id) 

183 download_youtube_video(yt_tutorial_video, tutorial_id, videos_output_folder) 

184 # Full video 

185 logging.info("Will now download the full video %s..." % full_video_id) 

186 # For the tutorial video we already had a Youtube object, not yet for the full video 

187 video_url = "https://www.youtube.com/watch?v=%s" % full_video_id 

188 yt_full_video = YouTube(video_url) 

189 download_youtube_video(yt_full_video, full_video_id, videos_output_folder) 

190 

191def download_youtube_video(yt, video_id, videos_output_folder): 

192 # Download stream with itag 18 by default: 

193 # <Stream: itag="18" mime_type="video/mp4" res="360p" fps="30fps" vcodec="avc1.42001E" acodec="mp4a.40.2"> 

194 stream = yt.streams.get_by_itag(18) 

195 if not stream: 

196 logging.debug("No stream available with itag 18") 

197 stream = yt.streams.filter(res = "360p", progressive=True, file_extension = "mp4").first() 

198 logging.debug("Stream that will be downloaded: %s" % stream) 

199 logging.debug("Download folder: %s" % videos_output_folder) 

200 try: 

201 download_stream(stream, videos_output_folder, video_id) 

202 except HTTPError as e: 

203 logging.error('An HTTP error %d occurred with reason: %s' % (e.code, e.reason)) 

204 # Propose to download that video manually from the browser 

205 webbrowser.open("https://y2mate.com/youtube/%s" % video_id, new=2, autoraise=True) 

206 return False 

207 return True 

208 

209def download_stream(stream, videos_output_folder, video_id): 

210 stream.download(videos_output_folder, video_id) 

211 

212def generate_new_tutorial_info(tutorial_slug, song_author, song_title, tutorial_id, full_video_id): 

213 new_tutorial_info = """{ 

214 "slug": "%s", 

215 "author": "%s", 

216 "title": "%s", 

217 "videos": [ 

218 {"id": "%s", "start": 0, "end": 999} 

219 ], 

220 "videos_full_tutorial": [], 

221 "full_version": "%s" 

222 }""" % (tutorial_slug, song_author, song_title, tutorial_id, full_video_id) 

223 return new_tutorial_info 

224 

225def update_tutoriales_data_file(tutoriales_data_file, new_tutorial_info): 

226 # Read in the tutoriales data file 

227 with open(tutoriales_data_file, 'r') as file : 

228 filedata = file.read() 

229 # Add the new tutorial info to the list of tutorials 

230 filedata = filedata.replace("}\n];", "},\n %s\n];" % new_tutorial_info) 

231 # Save edited file 

232 with open(tutoriales_data_file, 'w') as file: 

233 file.write(filedata) 

234 

235def index_new_tutorial_link(tutorial_slug, song_title, song_author): 

236 return """\n <div class="card mb-3" style="max-width: 17rem;"> 

237 <div class="card-body"> 

238 <h5 class="card-title">%s - %s</h5> 

239 <a href="%s" class="stretched-link text-hide">Ver el tutorial</a> 

240 </div> 

241 <div class="card-footer"><small class="text-muted">NNmNNs en NN partes</small></div> 

242 </div>""" % (song_title, song_author, tutorial_slug) 

243 

244def index_new_youtube_links(song_title, song_author, tutorial_url, tutocreator_channel, tutocreator): 

245 return '\n <li>%s - %s: <a href="%s">Tutorial en YouTube</a> por <a href="https://www.youtube.com/channel/%s">%s</a></li>' % (song_title, song_author, tutorial_url, tutocreator_channel, tutocreator) 

246 

247def dummy_index_update(tutorial_slug, song_title, song_author, tutorial_url, tutocreator_channel, tutocreator, output_folder): 

248 dummy_index_page = "%sindex-dummy.html" % output_folder 

249 logging.info("Creating a new dummy index page '%s' with links to be included later in the main index page." % dummy_index_page) 

250 filedata = index_new_tutorial_link(tutorial_slug, song_title, song_author) 

251 filedata += index_new_youtube_links(song_title, song_author, tutorial_url, tutocreator_channel, tutocreator) 

252 with open(dummy_index_page, 'w') as file : 

253 file.write(filedata) 

254 

255def dummy_symlink_files(output_folder): 

256 logging.debug("Creating symlinks for the .js and .css files in the dummy folder '%s'." % output_folder) 

257 os.symlink("../../vallenato.fr.js", "%svallenato.fr.js" % output_folder) 

258 os.symlink("../../vallenato.fr.css", "%svallenato.fr.css" % output_folder) 

259 

260def update_index_page(tutorial_slug, song_title, song_author, tutorial_url, tutocreator_channel, tutocreator): 

261 logging.info("Updating the index page with links to the new tutorial page.") 

262 # Read in the index page 

263 with open("../website/src/aprender/index.html", 'r') as file : 

264 filedata = file.read() 

265 

266 # Add a link to the new tutorial's page 

267 end_section = """ 

268 </div> 

269 </div> 

270 </div> 

271 <div class="row"> 

272 <div class="col-md"> 

273 <h2>Otros recursos</h2>""" 

274 new_link = index_new_tutorial_link(tutorial_slug, song_title, song_author) 

275 tut_number = filedata.count("<!-- Tutorial ") + 1 

276 #TODO: add "wrap every N on ZZ" depending on the tutorial's number 

277 filedata = filedata.replace(end_section, "\n <!-- Tutorial %d -->%s\n%s" %(tut_number, new_link, end_section)) 

278 

279 # Add links to the tutorial and the author's YouTube channel 

280 end_section = '\n </ul>\n </div>\n </div>\n </div>\n </main>\n <!-- End page content -->' 

281 new_link = index_new_youtube_links(song_title, song_author, tutorial_url, tutocreator_channel, tutocreator) 

282 filedata = filedata.replace(end_section, "%s%s" %(new_link, end_section)) 

283 

284 # Save edited file 

285 with open("../website/src/aprender/index.html", 'w') as file: 

286 file.write(filedata) 

287 

288def aprender(args): 

289 # Get the information about this new tutorial 

290 (tutorial_id, tutorial_url, full_video_id, full_video_url, song_title, song_author, tutocreator, tutocreator_channel, yt_tutorial_video, tutorial_slug) = get_tutorial_info() 

291 

292 # Determine the output folder (depends on the --temp-folder parameter) 

293 output_folder = determine_output_folder(args.temp_folder, tutorial_slug) 

294 

295 # Get the info that will be added for the new tutorial 

296 new_tutorial_info = generate_new_tutorial_info(tutorial_slug, song_author, song_title, tutorial_id, full_video_id) 

297 

298 if args.temp_folder: 

299 # When creating the new tutorial in a temporary folder for later edition, do not update the index page 

300 dummy_index_update(tutorial_slug, song_title, song_author, tutorial_url, tutocreator_channel, tutocreator, output_folder) 

301 # Symlink files so that the new template can be used from the temp folder 

302 dummy_symlink_files(output_folder) 

303 else: 

304 # Update the index page with the links to the new tutorial and to the tuto's author page 

305 update_index_page(tutorial_slug, song_title, song_author, tutorial_url, tutocreator_channel, tutocreator) 

306 # Add the new tutorial to the list of tutorials 

307 update_tutoriales_data_file(TUTORIALES_DATA_FILE, new_tutorial_info) 

308 

309 # Download the videos (both the tutorial and the full video) 

310 if args.no_download: 

311 logging.info("Not downloading the videos from YouTube due to --no-download parameter.") 

312 else: 

313 videos_output_folder = "%svideos/%s/" % (output_folder, tutorial_slug) 

314 if not os.path.exists(videos_output_folder): 

315 logging.debug("Creating folder '%s'." % videos_output_folder) 

316 os.makedirs(videos_output_folder) 

317 download_videos(yt_tutorial_video, tutorial_id, full_video_id, videos_output_folder) 

318 

319 # Open the new tutorial page in the webbrowser (new tab) for edition 

320 new_tutorial_page = "http://localhost:8000/aprender/?new_tutorial=%s" % tutorial_slug 

321 logging.debug("Opening new tab in web browser to '%s'" % new_tutorial_page) 

322 webbrowser.open(new_tutorial_page, new=2, autoraise=True)