tech234a
/
ytcc-archive
mirror of https://github.com/Data-Horde/ytcc-archive
1
0
Fork 0
Code Issues 0 Releases 0 Wiki Activity
archiving community contributions on YouTube: unpublished captions, title and description translations and caption credits
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
141 Commits
7 Branches
162 KiB
 
 
Tree: 2db5e71e87
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from '2db5e71e87'
${ noResults }
ytcc-archive/worker.py

346 lines
22 KiB
Raw Blame History 

  1. from threading import Thread

  2. import requests

  3. from time import sleep

  4. from os import mkdir, rmdir, listdir, system, environ

  5. from os.path import isdir, isfile, getsize

  6. from json import dumps, loads

  7. 

  8. from youtube_channel import main

  9. 

  10. import signal

  11. 

  12. import tracker

  13. 

  14. from youtube_dl import YoutubeDL

  15. 

  16. from shutil import rmtree, which

  17. 

  18. from queue import Queue

  19. 

  20. from gc import collect

  21. 

  22. from discovery import getmetadata

  23. from export import subprrun

  24. 

  25. #useful Queue example: https://stackoverflow.com/a/54658363

  26. jobs = Queue()

  27. 

  28. try:

  29.     mkdir("out")

  30. except:

  31.     pass

  32. 

  33. try:

  34.     mkdir("directory")

  35. except:

  36.     pass

  37. 

  38. HEROKU = False

  39. if isfile("../Procfile"):

  40.     HEROKU = True

  41. 

  42. langs = ['ab', 'aa', 'af', 'sq', 'ase', 'am', 'ar', 'arc', 'hy', 'as', 'ay', 'az', 'bn', 'ba', 'eu', 'be', 'bh', 'bi', 'bs', 'br', 

  43.     'bg', 'yue', 'yue-HK', 'ca', 'chr', 'zh-CN', 'zh-HK', 'zh-Hans', 'zh-SG', 'zh-TW', 'zh-Hant', 'cho', 'co', 'hr', 'cs', 'da', 'nl', 

  44.     'nl-BE', 'nl-NL', 'dz', 'en', 'en-CA', 'en-IN', 'en-IE', 'en-GB', 'en-US', 'eo', 'et', 'fo', 'fj', 'fil', 'fi', 'fr', 'fr-BE', 

  45.     'fr-CA', 'fr-FR', 'fr-CH', 'ff', 'gl', 'ka', 'de', 'de-AT', 'de-DE', 'de-CH', 'el', 'kl', 'gn', 'gu', 'ht', 'hak', 'hak-TW', 'ha', 

  46.     'iw', 'hi', 'hi-Latn', 'ho', 'hu', 'is', 'ig', 'id', 'ia', 'ie', 'iu', 'ik', 'ga', 'it', 'ja', 'jv', 'kn', 'ks', 'kk', 'km', 'rw', 

  47.     'tlh', 'ko', 'ku', 'ky', 'lo', 'la', 'lv', 'ln', 'lt', 'lb', 'mk', 'mg', 'ms', 'ml', 'mt', 'mni', 'mi', 'mr', 'mas', 'nan', 

  48.     'nan-TW', 'lus', 'mo', 'mn', 'my', 'na', 'nv', 'ne', 'no', 'oc', 'or', 'om', 'ps', 'fa', 'fa-AF', 'fa-IR', 'pl', 'pt', 'pt-BR', 

  49.     'pt-PT', 'pa', 'qu', 'ro', 'rm', 'rn', 'ru', 'ru-Latn', 'sm', 'sg', 'sa', 'sc', 'gd', 'sr', 'sr-Cyrl', 'sr-Latn', 'sh', 'sdp', 'sn', 

  50.     'scn', 'sd', 'si', 'sk', 'sl', 'so', 'st', 'es', 'es-419', 'es-MX', 'es-ES', 'es-US', 'su', 'sw', 'ss', 'sv', 'tl', 'tg', 'ta', 

  51.     'tt', 'te', 'th', 'bo', 'ti', 'tpi', 'to', 'ts', 'tn', 'tr', 'tk', 'tw', 'uk', 'ur', 'uz', 'vi', 'vo', 'vor', 'cy', 'fy', 'wo', 

  52.     'xh', 'yi', 'yo', 'zu']

  53. 

  54. assert which("zip") and which("rsync") and which("curl"), "Please ensure the zip, rsync, and curl commands are installed on your system."

  55. 

  56. #HSID, SSID, SID cookies required

  57. if "HSID" in environ.keys() and "SSID" in environ.keys() and "SID" in environ.keys():

  58.     cookies = {"HSID": environ["HSID"], "SSID": environ["SSID"], "SID": environ["SID"]}

  59. elif isfile("config.json"):

  60.     cookies = loads(open("config.json").read())

  61. else:

  62.     print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  63.     assert False

  64. if not (cookies["HSID"] and cookies["SSID"] and cookies["SID"]):

  65.     print("HSID, SSID, and SID cookies from youtube.com are required. Specify in config.json or as environment variables.")

  66.     assert False

  67. 

  68. mysession = requests.session()

  69. mysession.headers.update({"cookie": "HSID="+cookies["HSID"]+"; SSID="+cookies["SSID"]+"; SID="+cookies["SID"], "Accept-Language": "en-US",})

  70. 

  71. validationtest = mysession.get("https://www.youtube.com/timedtext_editor?action_mde_edit_form=1&v=1iNTtHUwvq4&lang=en&bl=vmp&ui=hd&ref=player&tab=captions&o=U")

  72. 

  73. assert not "accounts.google.com" in validationtest.url, "Please ensure you have correctly specified account cookies."

  74. assert """<button class="yt-uix-button yt-uix-button-size-default yt-uix-button-default yt-uix-button-has-icon" type="button" onclick=";return false;" id="yt-picker-language-button" data-button-action="yt.www.picker.load" data-button-menu-id="arrow-display" data-picker-key="language" data-picker-position="footer" data-button-toggle="true"><span class="yt-uix-button-icon-wrapper"><span class="yt-uix-button-icon yt-uix-button-icon-footer-language yt-sprite"></span></span><span class="yt-uix-button-content">  <span class="yt-picker-button-label">

  75. Language:

  76.   </span>

  77.   English

  78. </span><span class="yt-uix-button-arrow yt-sprite"></span></button>""" in validationtest.text, "Please make sure your YouTube and Google account language is set to English (United States)"

  79. 

  80. del validationtest

  81. 

  82. open("cookies.txt", "w").write("""# HTTP Cookie File

  83. .youtube.com	TRUE	/	FALSE	1663793455	SID	[SID]

  84. .youtube.com	TRUE	/	FALSE	1663793455	HSID	[HSID]

  85. .youtube.com	TRUE	/	TRUE	1663793455	SSID	[SSID]""".replace("[SID]", cookies["SID"]).replace("[HSID]", cookies["HSID"]).replace("[SSID]", cookies["SSID"]))

  86. 

  87. del cookies

  88. 

  89. validationtimes = 0

  90. shouldgetjob = True

  91. 

  92. #Graceful Shutdown

  93. class GracefulKiller:

  94.     kill_now = False

  95.     def __init__(self):

  96.         signal.signal(signal.SIGINT, self.exit_gracefully)

  97.         signal.signal(signal.SIGTERM, self.exit_gracefully)

  98. 

  99.     def exit_gracefully(self, signum, frame):

  100.         print("Graceful exit process initiated, no longer accepting new tasks but finishing existing ones...")

  101.         self.kill_now = True

  102. 

  103. gkiller = GracefulKiller()

  104. 

  105. #REMOVED PANIC MECHANISM!

  106. """

  107. enres = getmetadata(mysession, "IjJKfe-0Ty0", True)[0]

  108. if not enres:

  109.     print("Community Contribution discovery has been disabled for this account, please report this on our Discord as this may have caused some videos to be incorrectly marked as having community contributions disabled.")

  110.     shouldgetjob = False

  111.     gkiller.kill_now = True #exit the script

  112. 

  113. del enres

  114. """

  115. 

  116. #microtasks

  117. def threadrunner():

  118.     global shouldgetjob

  119.     global validationtimes

  120.     jobs = Queue()

  121.     ydl = YoutubeDL({"extract_flat": "in_playlist", "simulate": True, "skip_download": True, "quiet": True, "cookiefile": "cookies.txt", "source_address": "0.0.0.0", "call_home": False})

  122.     while True:

  123.         if not jobs.empty():

  124.             task, vid, args = jobs.get()

  125.             if task == "submitdiscovery":

  126.                 tracker.add_item_to_tracker(args, vid)

  127.                 #pass

  128.             elif task == "discovery":

  129.                 

  130.                 while True:

  131.                     try:

  132.                         info = getmetadata(mysession, str(vid).strip())

  133.                         break

  134.                     except BaseException as e:

  135.                         print(e)

  136.                         print("Error in retrieving information, waiting 30 seconds and trying again")

  137.                         sleep(30)

  138.                 if info[0]: # ccenabled or creditdata

  139.                     if not isdir("out/"+str(vid).strip()):

  140.                         mkdir("out/"+str(vid).strip())

  141. 

  142.                 if info[0]:

  143.                     for langcode in langs:

  144.                         jobs.put(("subtitles", vid, langcode))

  145. 

  146.                     for langcode in langs:

  147.                         jobs.put(("subtitles-forceedit-metadata", vid, langcode))

  148. 

  149.                     for langcode in langs:

  150.                         jobs.put(("subtitles-forceedit-captions", vid, langcode))

  151. 

  152.                 jobs.put(("complete", None, "video:"+vid))

  153. 

  154.                 for videodisc in info[1]:

  155.                     jobs.put(("submitdiscovery", videodisc, tracker.ItemType.Video))

  156.                 for channeldisc in info[2]:

  157.                     jobs.put(("submitdiscovery", channeldisc, tracker.ItemType.Channel))

  158.                 for mixdisc in info[3]:

  159.                     jobs.put(("submitdiscovery", mixdisc, tracker.ItemType.MixPlaylist))

  160.                 for playldisc in info[4]:

  161.                     jobs.put(("submitdiscovery", playldisc, tracker.ItemType.Playlist))

  162.                 

  163.                 #jobs.put(("complete", None, "video:"+vid))

  164.                 #pass

  165. 

  166.             elif task == "subtitles":

  167.                 subprrun(mysession, args, vid, "default", needforcemetadata, needforcecaptions)

  168.             elif task == "subtitles-forceedit-captions":

  169.                 subprrun(mysession, args, vid, "forceedit-captions", needforcemetadata, needforcecaptions)

  170.             elif task == "subtitles-forceedit-metadata":

  171.                 subprrun(mysession, args, vid, "forceedit-metadata", needforcemetadata, needforcecaptions)

  172.             elif task == "channel":

  173.                 try:

  174.                     y = ydl.extract_info("https://www.youtube.com/channel/"+desit.split(":", 1)[1], download=False)

  175.                     for itemyv in y["entries"]:

  176.                         jobs.put(("submitdiscovery", itemyv["id"], tracker.ItemType.Video))

  177. 

  178.                     #channel created playlists

  179.                     y = main(desit.split(":", 1)[1])

  180.                     for itemyv in y["playlists"]:

  181.                         jobs.put(("submitdiscovery", itemyv, tracker.ItemType.Playlist))

  182.                     for itemyv in y["channels"]:

  183.                         jobs.put(("submitdiscovery", itemyv, tracker.ItemType.Channel))

  184. 

  185.                     jobs.put(("complete", None, "channel:"+args))

  186.                 except:

  187.                     print("YouTube-DL error, ignoring but not marking as complete...", "https://www.youtube.com/channel/"+desit.split(":", 1)[1])

  188.             elif task == "playlist":

  189.                 try:

  190.                     y = ydl.extract_info("https://www.youtube.com/playlist?list="+desit.split(":", 1)[1], download=False)

  191.                     #TODO: extract owner channel in other projects

  192.                     #TODO: handle channels in other projects, not needed here because we will get it from the video

  193.                     for itemyvp in y["entries"]:

  194.                         jobs.put(("submitdiscovery", itemyvp["id"], tracker.ItemType.Video))

  195.                     jobs.put(("complete", None, "playlist:"+args))

  196.                 except:

  197.                     print("YouTube-DL error, ignoring but not marking as complete...", "https://www.youtube.com/playlist?list="+desit.split(":", 1)[1])

  198.             elif task == "mixplaylist":

  199.                 try:

  200.                     wptext = mysession.get("https://www.youtube.com/watch?v=jNQXAC9IVRw&list="+desit.split(":", 1)[1]).text

  201.                     #chanl = set()

  202.                     #channel handling not needed here because we will get it from the video

  203.                     for line in wptext.splitlines():

  204.                         if line.strip().startswith('window["ytInitialData"] = '):

  205.                             initdata = loads(line.split('window["ytInitialData"] = ', 1)[1].strip()[:-1])

  206.                             for itemyvp in initdata["contents"]["twoColumnWatchNextResults"]["playlist"]["playlist"]["contents"]:

  207.                                 jobs.put(("submitdiscovery", itemyvp["playlistPanelVideoRenderer"]["videoId"], tracker.ItemType.Video))

  208.                                 #chanl.add(itemyvp["playlistPanelVideoRenderer"]["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"])

  209.                                 

  210.                     #for itemn in chanl:

  211.                     #    jobs.put(("submitdiscovery", itemn, tracker.ItemType.Channel))

  212.                     jobs.put(("complete", None, "mixplaylist:"+args))

  213.                 except:

  214.                     print("Mix Playlist error, ignoring but not marking as complete...", "https://www.youtube.com/watch?v=jNQXAC9IVRw&list="+desit.split(":", 1)[1])

  215.             elif task == "complete":

  216.                 size = 0

  217.                 if ":" in args:

  218.                     if args.split(":", 1)[0] == "video":

  219.                         #check if dir is empty, make zip if needed

  220.                         if isdir("out/"+args.split(":", 1)[1]):

  221.                             if not listdir("out/"+args.split(":", 1)[1]):

  222.                                 rmdir("out/"+args.split(":", 1)[1])

  223.                             else:

  224.                                 #zip it up

  225.                                 if not isdir("directory/"+args.split(":", 1)[1]):

  226.                                     mkdir("directory/"+args.split(":", 1)[1])

  227. 

  228.                                 while not isfile("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip"):

  229.                                     print("Attempting to zip item...")

  230.                                     system("zip -9 -r -j directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip out/"+args.split(":", 1)[1])

  231. 

  232.                                 #get a target

  233.                                 targetloc = None

  234.                                 while not targetloc:

  235.                                     targetloc = tracker.request_upload_target()

  236.                                     if targetloc:

  237.                                         break

  238.                                     else:

  239.                                         print("Waiting 5 minutes...")

  240.                                         sleep(300)

  241. 

  242.                                 if targetloc.startswith("rsync"):

  243.                                     system("rsync -rltv --timeout=300 --contimeout=300 --progress --bwlimit 0 --recursive --partial --partial-dir .rsync-tmp --min-size 1 --no-compress --compress-level 0 directory/"+args.split(":", 1)[1]+"/ "+targetloc)

  244.                                 elif targetloc.startswith("http"):

  245.                                     system("curl -F "+args.split(":", 1)[1]+".zip=@directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip "+targetloc)

  246. 

  247. 

  248.                                 size = getsize("directory/"+args.split(":", 1)[1]+"/"+args.split(":", 1)[1]+".zip")

  249.                                 #cleanup

  250.                                 try:

  251.                                     del langcnt[args.split(":", 1)[1]]

  252.                                     rmtree("directory/"+args.split(":", 1)[1]+"/")

  253.                                     rmdir("directory/"+args.split(":", 1)[1]+"/")

  254.                                     rmtree("out/"+args.split(":", 1)[1]+"/")

  255.                                     rmdir("out/"+args.split(":", 1)[1]+"/")

  256.                                 except:

  257.                                     pass

  258.                 tracker.mark_item_as_done(args, size)

  259.             jobs.task_done()

  260.         else:

  261.             if not gkiller.kill_now:

  262.                 # get a new task from tracker

  263.                 collect() #cleanup

  264. 

  265.                 #Protection Mechanism Disarmed

  266.                 """

  267.                 #check that the account has community contributions enabled every 50th item

  268.                 validationtimes += 1

  269.                 if not validationtimes % 50:

  270.                     enres = getmetadata(mysession, "IjJKfe-0Ty0", True)[0]

  271.                     if not enres:

  272.                         print("Community Contribution discovery has been disabled for this account, please report this on our Discord as this may have caused some videos to be incorrectly marked as having community contributions disabled.")

  273.                         shouldgetjob = False

  274.                         gkiller.kill_now = True #exit the script

  275.                     del enres

  276.                 """

  277. 

  278.                 if shouldgetjob:

  279.                     desit = tracker.request_item_from_tracker()

  280.                     print("New task:", desit)

  281. 

  282.                     if desit:

  283.                         if desit.split(":", 1)[0] == "video":

  284.                             needforcemetadata = {'ab': None, 'aa': None, 'af': None, 'sq': None, 'ase': None, 'am': None, 'ar': None, 'arc': None, 'hy': None, 'as': None, 'ay': None, 'az': None, 'bn': None, 'ba': None, 'eu': None, 'be': None, 'bh': None, 'bi': None, 'bs': None, 'br': None, 

  285.                                 'bg': None, 'yue': None, 'yue-HK': None, 'ca': None, 'chr': None, 'zh-CN': None, 'zh-HK': None, 'zh-Hans': None, 'zh-SG': None, 'zh-TW': None, 'zh-Hant': None, 'cho': None, 'co': None, 'hr': None, 'cs': None, 'da': None, 'nl': None, 

  286.                                 'nl-BE': None, 'nl-NL': None, 'dz': None, 'en': None, 'en-CA': None, 'en-IN': None, 'en-IE': None, 'en-GB': None, 'en-US': None, 'eo': None, 'et': None, 'fo': None, 'fj': None, 'fil': None, 'fi': None, 'fr': None, 'fr-BE': None, 

  287.                                 'fr-CA': None, 'fr-FR': None, 'fr-CH': None, 'ff': None, 'gl': None, 'ka': None, 'de': None, 'de-AT': None, 'de-DE': None, 'de-CH': None, 'el': None, 'kl': None, 'gn': None, 'gu': None, 'ht': None, 'hak': None, 'hak-TW': None, 'ha': None, 

  288.                                 'iw': None, 'hi': None, 'hi-Latn': None, 'ho': None, 'hu': None, 'is': None, 'ig': None, 'id': None, 'ia': None, 'ie': None, 'iu': None, 'ik': None, 'ga': None, 'it': None, 'ja': None, 'jv': None, 'kn': None, 'ks': None, 'kk': None, 'km': None, 'rw': None, 

  289.                                 'tlh': None, 'ko': None, 'ku': None, 'ky': None, 'lo': None, 'la': None, 'lv': None, 'ln': None, 'lt': None, 'lb': None, 'mk': None, 'mg': None, 'ms': None, 'ml': None, 'mt': None, 'mni': None, 'mi': None, 'mr': None, 'mas': None, 'nan': None, 

  290.                                 'nan-TW': None, 'lus': None, 'mo': None, 'mn': None, 'my': None, 'na': None, 'nv': None, 'ne': None, 'no': None, 'oc': None, 'or': None, 'om': None, 'ps': None, 'fa': None, 'fa-AF': None, 'fa-IR': None, 'pl': None, 'pt': None, 'pt-BR': None, 

  291.                                 'pt-PT': None, 'pa': None, 'qu': None, 'ro': None, 'rm': None, 'rn': None, 'ru': None, 'ru-Latn': None, 'sm': None, 'sg': None, 'sa': None, 'sc': None, 'gd': None, 'sr': None, 'sr-Cyrl': None, 'sr-Latn': None, 'sh': None, 'sdp': None, 'sn': None, 

  292.                                 'scn': None, 'sd': None, 'si': None, 'sk': None, 'sl': None, 'so': None, 'st': None, 'es': None, 'es-419': None, 'es-MX': None, 'es-ES': None, 'es-US': None, 'su': None, 'sw': None, 'ss': None, 'sv': None, 'tl': None, 'tg': None, 'ta': None, 

  293.                                 'tt': None, 'te': None, 'th': None, 'bo': None, 'ti': None, 'tpi': None, 'to': None, 'ts': None, 'tn': None, 'tr': None, 'tk': None, 'tw': None, 'uk': None, 'ur': None, 'uz': None, 'vi': None, 'vo': None, 'vor': None, 'cy': None, 'fy': None, 'wo': None, 

  294.                                 'xh': None, 'yi': None, 'yo': None, 'zu': None}

  295.                             needforcecaptions = {'ab': None, 'aa': None, 'af': None, 'sq': None, 'ase': None, 'am': None, 'ar': None, 'arc': None, 'hy': None, 'as': None, 'ay': None, 'az': None, 'bn': None, 'ba': None, 'eu': None, 'be': None, 'bh': None, 'bi': None, 'bs': None, 'br': None, 

  296.                                 'bg': None, 'yue': None, 'yue-HK': None, 'ca': None, 'chr': None, 'zh-CN': None, 'zh-HK': None, 'zh-Hans': None, 'zh-SG': None, 'zh-TW': None, 'zh-Hant': None, 'cho': None, 'co': None, 'hr': None, 'cs': None, 'da': None, 'nl': None, 

  297.                                 'nl-BE': None, 'nl-NL': None, 'dz': None, 'en': None, 'en-CA': None, 'en-IN': None, 'en-IE': None, 'en-GB': None, 'en-US': None, 'eo': None, 'et': None, 'fo': None, 'fj': None, 'fil': None, 'fi': None, 'fr': None, 'fr-BE': None, 

  298.                                 'fr-CA': None, 'fr-FR': None, 'fr-CH': None, 'ff': None, 'gl': None, 'ka': None, 'de': None, 'de-AT': None, 'de-DE': None, 'de-CH': None, 'el': None, 'kl': None, 'gn': None, 'gu': None, 'ht': None, 'hak': None, 'hak-TW': None, 'ha': None, 

  299.                                 'iw': None, 'hi': None, 'hi-Latn': None, 'ho': None, 'hu': None, 'is': None, 'ig': None, 'id': None, 'ia': None, 'ie': None, 'iu': None, 'ik': None, 'ga': None, 'it': None, 'ja': None, 'jv': None, 'kn': None, 'ks': None, 'kk': None, 'km': None, 'rw': None, 

  300.                                 'tlh': None, 'ko': None, 'ku': None, 'ky': None, 'lo': None, 'la': None, 'lv': None, 'ln': None, 'lt': None, 'lb': None, 'mk': None, 'mg': None, 'ms': None, 'ml': None, 'mt': None, 'mni': None, 'mi': None, 'mr': None, 'mas': None, 'nan': None, 

  301.                                 'nan-TW': None, 'lus': None, 'mo': None, 'mn': None, 'my': None, 'na': None, 'nv': None, 'ne': None, 'no': None, 'oc': None, 'or': None, 'om': None, 'ps': None, 'fa': None, 'fa-AF': None, 'fa-IR': None, 'pl': None, 'pt': None, 'pt-BR': None, 

  302.                                 'pt-PT': None, 'pa': None, 'qu': None, 'ro': None, 'rm': None, 'rn': None, 'ru': None, 'ru-Latn': None, 'sm': None, 'sg': None, 'sa': None, 'sc': None, 'gd': None, 'sr': None, 'sr-Cyrl': None, 'sr-Latn': None, 'sh': None, 'sdp': None, 'sn': None, 

  303.                                 'scn': None, 'sd': None, 'si': None, 'sk': None, 'sl': None, 'so': None, 'st': None, 'es': None, 'es-419': None, 'es-MX': None, 'es-ES': None, 'es-US': None, 'su': None, 'sw': None, 'ss': None, 'sv': None, 'tl': None, 'tg': None, 'ta': None, 

  304.                                 'tt': None, 'te': None, 'th': None, 'bo': None, 'ti': None, 'tpi': None, 'to': None, 'ts': None, 'tn': None, 'tr': None, 'tk': None, 'tw': None, 'uk': None, 'ur': None, 'uz': None, 'vi': None, 'vo': None, 'vor': None, 'cy': None, 'fy': None, 'wo': None, 

  305.                                 'xh': None, 'yi': None, 'yo': None, 'zu': None}

  306.                             jobs.put(("discovery", desit.split(":", 1)[1], None))

  307.                         elif desit.split(":", 1)[0] == "channel":

  308.                             jobs.put(("channel", None, desit.split(":", 1)[1]))

  309.                         elif desit.split(":", 1)[0] == "playlist":

  310.                             jobs.put(("playlist", None, desit.split(":", 1)[1]))

  311.                         elif desit.split(":", 1)[0] == "mixplaylist":

  312.                             jobs.put(("mixplaylist", None, desit.split(":", 1)[1]))

  313.                         else:

  314.                             print("Ignoring item for now", desit)

  315.                     else:

  316.                         print("Ignoring item for now", desit)

  317.                 else:

  318.                     break

  319.             else:

  320.                 break

  321.     

  322. 

  323. threads = []

  324. 

  325. THREADCNT = 50

  326. if HEROKU:

  327.     THREADCNT = 20

  328. #now create the rest of the threads

  329. for i in range(THREADCNT):

  330.     runthread = Thread(target=threadrunner)

  331.     runthread.start()

  332.     threads.append(runthread)

  333.     del runthread

  334. 

  335. #https://stackoverflow.com/a/11968881

  336. for x in threads:

  337.     x.join()

  338.     threads.remove(x)

  339.     del x

  340. 

  341. if not shouldgetjob:

  342.     print("PROTECTION MECHANISM #3 WAS SOMEHOW TRIGERRED")

  343.     print("Community Contribution discovery has been disabled for this account, please report this on our Discord as this may have caused some videos to be incorrectly marked as having community contributions disabled.")

  344. 

  345. print("Exiting...")