tech234a
/
ytcc-archive
Mirror von https://github.com/Data-Horde/ytcc-archive
1
0
Fork 0
Code Issues 0 Releases 0 Wiki Aktivität
archiving community contributions on YouTube: unpublished captions, title and description translations and caption credits
Du kannst nicht mehr als 25 Themen auswählen Themen müssen entweder mit einem Buchstaben oder einer Ziffer beginnen. Sie können Bindestriche („-“) enthalten und bis zu 35 Zeichen lang sein.
71 Commits
7 Branches
218 KiB
 
 
Struktur: d125887b58
Branches Tags
${ item.name }
Erstelle Branch ${ searchTerm }
von „d125887b58“
${ noResults }
ytcc-archive/discovery.py

132 Zeilen
11 KiB
Originalformat Blame Verlauf 

  1. from typing import Dict

  2. import requests

  3. from json import loads

  4. 

  5. langcodes = {"Afar": "aa", "Abkhazian": "ab", "Afrikaans": "af", "Akan": "ak", "all": "all", "Amharic": "am", "Aragonese": "an", "Arabic": "ar", "Aramaic": "arc", "Algerian Arabic": "arq", "Assamese": "as", "American Sign Language": "ase", "Asturian": "ast", "Avaric": "av", "Aymara": "ay", "Azerbaijani": "az", "Bashkir": "ba", "Belarusian": "be", "Bulgarian": "bg", "Bihari": "bh", "Bislama": "bi", "Bangla": "bn", "Tibetan": "bo", "Breton": "br", "Bosnian": "bs", "Catalan": "ca", "Cebuano": "ceb", "Choctaw": "cho", "Cherokee": "chr", "Corsican": "co", "Czech": "cs", "Church Slavic": "cu", "Welsh": "cy", "Danish": "da", "Danish (Denmark)": "da-DK", "German": "de", "German (Austria)": "de-AT", "German (Switzerland)": "de-CH", "German (Germany)": "de-DE", "Divehi": "dv", "Dzongkha": "dz", "Ewe": "ee", "Greek": "el", "English": "en", "English (United Arab Emirates)": "en-AE", "English (Canada)": "en-CA", "English (United Kingdom)": "en-GB", "English (Ireland)": "en-IE", "English (India)": "en-IN", "English (United States)": "en-US", "Esperanto": "eo", "Spanish": "es", "Spanish (Latin America)": "es-419", "Spanish (Argentina)": "es-AR", "Spanish (Chile)": "es-CL", "Spanish (Colombia)": "es-CO", "Spanish (Costa Rica)": "es-CR", "Spanish (Spain)": "es-ES", "Spanish (Mexico)": "es-MX", "Spanish (Nicaragua)": "es-NI", "Spanish (United States)": "es-US", "Estonian": "et", "Basque": "eu", "Persian": "fa", "Persian (Afghanistan)": "fa-AF", "Persian (Iran)": "fa-IR", "Fulah": "ff", "Finnish": "fi", "Filipino": "fil", "Fijian": "fj", "Faroese": "fo", "French": "fr", "French (Belgium)": "fr-BE", "French (Canada)": "fr-CA", "French (Switzerland)": "fr-CH", "French (France)": "fr-FR", "Western Frisian": "fy", "Irish": "ga", "Scottish Gaelic": "gd", "Galician": "gl", "Guarani": "gn", "Swiss German": "gsw", "Gujarati": "gu", "Hausa": "ha", "Hakka Chinese": "hak", "Hakka Chinese (Taiwan)": "hak-TW", "Hindi": "hi-Latn", "Hmong": "hmn", "Croatian": "hr", "Haitian Creole": "ht", "Hungarian": "hu", "Armenian": "hy", "Interlingua": "ia", "Indonesian": "id", "Interlingue": "ie", "Igbo": "ig", "Sichuan Yi": "ii", "Inupiaq": "ik", "Icelandic": "is", "Italian": "it", "Italian (Italy)": "it-IT", "Inuktitut": "iu", "Hebrew": "iw", "Japanese": "ja", "Javanese": "jv", "Georgian": "ka", "Kazakh": "kk", "Kalaallisut": "kl", "Khmer": "km", "Kannada": "kn", "Korean": "ko", "Korean (South Korea)": "ko-KR", "Kanuri": "kr", "Kashmiri": "ks", "Kurdish": "ku", "Kyrgyz": "ky", "Latin": "la", "Luxembourgish": "lb", "Lingala": "ln", "Lao": "lo", "Lithuanian": "lt", "Mizo": "lus", "Latvian": "lv", "Masai": "mas", "Malagasy": "mg", "Maori": "mi", "Miscellaneous languages": "mis", "Macedonian": "mk", "Malayalam": "ml", "Mongolian": "mn", "Manipuri": "mni", "Moldavian": "mo", "Marathi": "mr", "Malay": "ms", "Maltese": "mt", "Burmese": "my", "Nauru": "na", "Min Nan Chinese": "nan", "Min Nan Chinese (Taiwan)": "nan-TW", "Nepali": "ne", "Dutch": "nl", "Dutch (Belgium)": "nl-BE", "Dutch (Netherlands)": "nl-NL", "Norwegian Nynorsk": "nn", "Norwegian": "no", "not": "not", "Navajo": "nv", "Occitan": "oc", "Oromo": "om", "Odia": "or", "Punjabi": "pa", "Polish": "pl", "Polish (Poland)": "pl-PL", "Pashto": "ps", "Portuguese": "pt", "Portuguese (Brazil)": "pt-BR", "Portuguese (Portugal)": "pt-PT", "Quechua": "qu", "Romansh": "rm", "Rundi": "rn", "Romanian": "ro", "Romanian (Moldova)": "ro-MD", "Russian": "ru-Latn", "Russian (Russia)": "ru-RU", "Kinyarwanda": "rw", "Sanskrit": "sa", "Sardinian": "sc", "Sicilian": "scn", "Scots": "sco", "Sindhi": "sd", "Sherdukpen": "sdp", "Northern Sami": "se", "Sango": "sg", "Serbo-Croatian": "sh", "Sinhala": "si", "Slovak": "sk", "Slovenian": "sl", "Samoan": "sm", "Shona": "sn", "Somali": "so", "Albanian": "sq", "Serbian": "sr", "Serbian (Cyrillic)": "sr-Cyrl", "Serbian (Latin)": "sr-Latn", "Swati": "ss", "Southern Sotho": "st", "Sundanese": "su", "Swedish": "sv", "Swahili": "sw", "Tamil": "ta", "Telugu": "te", "Tajik": "tg", "Thai": "th", "Tigrinya": "ti", "Turkmen": "tk", "Tagalog": "tl", "Klingon": "tlh", "Tswana": "tn", "Tongan": "to", "Turkish": "tr", "Turkish (Turkey)": "tr-TR", "Tsonga": "ts", "Tatar": "tt", "Twi": "tw", "Ukrainian": "uk", "Urdu": "ur", "Uzbek": "uz", "Vietnamese": "vi", "Volap\\xFCk": "vo", "Wolof": "wo", "Xhosa": "xh", "Yiddish": "yi", "Yoruba": "yo", "Cantonese": "yue", "Cantonese (Hong Kong)": "yue-HK", "Chinese": "zh", "Chinese (China)": "zh-CN", "Chinese (Hong Kong)": "zh-HK", "Chinese (Simplified)": "zh-Hans", "Chinese (Simplified, China)": "zh-Hans-CN", "Chinese (Simplified, Singapore)": "zh-Hans-SG", "Chinese (Traditional)": "zh-Hant", "Chinese (Traditional, Hong Kong)": "zh-Hant-HK", "Chinese (Traditional, Taiwan)": "zh-Hant-TW", "Chinese (Singapore)": "zh-SG", "Chinese (Taiwan)": "zh-TW", "Zulu": "zu", "Hiri Motu": "ho", "Tok Pisin": "tpi", "Voro": "vor"}

  6. 

  7. def getmetadata(vid):

  8.     params = (

  9.         ("v", vid),

  10.     )

  11. 

  12.     headers = {

  13.         "Accept-Language": "en-US",

  14.     }

  15. 

  16.     wpage = requests.get("https://www.youtube.com/watch", headers=headers, params=params)

  17. 

  18.     wptext = wpage.text

  19. 

  20.     initplay = None

  21.     initdata = None

  22. 

  23.     recvids  = set()

  24.     recchans = set()

  25.     recmixes = set()

  26.     recplayl = set()

  27. 

  28.     ccenabled = False #default values

  29.     creditdata = {}

  30. 

  31.     for line in wptext.splitlines():

  32.         if line.strip().startswith('window["ytInitialPlayerResponse"] = '):

  33.             initplay = loads(line.split('window["ytInitialPlayerResponse"] = ', 1)[1].strip()[:-1])

  34. 

  35.             if initplay["playabilityStatus"]["status"] == "ERROR":

  36.                 print(vid, "unavailable")

  37.                 return False, {}, recvids, recchans, recmixes, recplayl

  38.             

  39.             if "endscreen" in initplay.keys():

  40.                 if "endscreenRenderer" in initplay["endscreen"].keys():

  41.                     for el in initplay["endscreen"]["endscreenRenderer"]:

  42. 

  43.                         if type(el) == Dict:

  44.                             elint = el["endscreenElementRenderer"]

  45. 

  46.                             if "endscreenElementRenderer" in el.keys():

  47.                                 if elint["style"] == "VIDEO":

  48.                                     recvids.add(elint["endpoint"]["watchEndpoint"]["videoId"])

  49. 

  50.                                 elif elint["style"] == "CHANNEL":

  51.                                     try:

  52.                                         recchans.add(elint["endpoint"]["browseEndpoint"]["browseId"])

  53.                                     except:

  54.                                         print("Channel endscreen error")

  55.                                         raise

  56. 

  57.                                 elif elint["style"] == "PLAYLIST":

  58.                                     recvids.add(elint["endpoint"]["watchEndpoint"]["videoId"])

  59.                                     recplayl.add(elint["endpoint"]["watchEndpint"]["playlistId"])

  60. 

  61.             if "captions" in initplay.keys():

  62.                 ccenabled = "contribute" in initplay["captions"]["playerCaptionsRenderer"]

  63.             else:

  64.                 ccenabled = False # if captions information is not present, community contributions are not enabled

  65. 

  66.             if "videoDetails" in initplay.keys():

  67.                 if "channelId" in initplay["videoDetails"].keys():

  68.                     recchans.add(initplay["videoDetails"]["channelId"])

  69.         elif line.strip().startswith('window["ytInitialData"] = '):

  70.             initdata = loads(line.split('window["ytInitialData"] = ', 1)[1].strip()[:-1])

  71.             if "contents" in initdata.keys(): #prevent exception

  72.                 for recmd in initdata["contents"]["twoColumnWatchNextResults"]["secondaryResults"]["secondaryResults"]["results"]:

  73.                     #auto is like the others

  74.                     if "compactAutoplayRenderer" in recmd.keys():

  75.                         recmd = recmd["compactAutoplayRenderer"]["contents"][0]

  76. 

  77.                     if "compactVideoRenderer" in recmd.keys():

  78.                         recvids.add(recmd["compactVideoRenderer"]["videoId"])

  79.                         try:

  80.                             recchans.add(recmd["compactVideoRenderer"]["channelId"])

  81.                         except KeyError as e:

  82.                             try:

  83.                                 recchans.add(recmd["compactVideoRenderer"]["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"])

  84.                             except KeyError as e:

  85.                                 print("Channel extract error")

  86.                             #raise

  87.                             #print("Unable to extract channel:")

  88.                             #print(recmd["compactVideoRenderer"])

  89. 

  90.                     elif "compactPlaylistRenderer" in recmd.keys():

  91.                         recplayl.add(recmd["compactPlaylistRenderer"]["playlistId"])

  92.                         if "navigationEndpoint" in recmd["compactPlaylistRenderer"].keys():

  93.                             recvids.add(recmd["compactPlaylistRenderer"]["navigationEndpoint"]["watchEndpoint"]["videoId"])

  94.                         if "navigationEndpoint" in recmd["compactPlaylistRenderer"]["shortBylineText"].keys():

  95.                             recchans.add(recmd["compactPlaylistRenderer"]["shortBylineText"]["navigationEndpoint"]["browseEndpoint"]["browseId"])

  96. 

  97.                     elif "compactRadioRenderer" in recmd.keys(): #mix playlist

  98.                         recmixes.add(recmd["compactRadioRenderer"]["playlistId"])

  99.                     # todo: find out if channels can be suggested

  100. 

  101.             creditdata = {}

  102.             try:

  103.                 mdinfo = initdata["contents"]["twoColumnWatchNextResults"]["results"]["results"]["contents"][1]["videoSecondaryInfoRenderer"]["metadataRowContainer"]["metadataRowContainerRenderer"]["rows"]

  104.                 for item in mdinfo:

  105.                     if item["metadataRowRenderer"]["title"]["simpleText"].startswith("Caption author"): #the request to /watch needs to be in English for this to work

  106.                         try:

  107.                             desl = langcodes[item["metadataRowRenderer"]["title"]["simpleText"].split("(", 1)[1][:-1]]

  108.                         except KeyError as e:

  109.                             #print(e)

  110.                             print("Language code conversion error, using language name")

  111.                             desl = item["metadataRowRenderer"]["title"]["simpleText"].split("(", 1)[1][:-1]

  112.                         creditdata[desl] = []

  113.                         for itemint in item["metadataRowRenderer"]["contents"]:

  114.                             creditdata[desl].append({"name": itemint["runs"][0]["text"], "channel": itemint["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]})

  115. 

  116.             except KeyError as e:

  117.                 #print("Video does not have credits")

  118.                 pass

  119.                 #raise

  120.                 #print(e)

  121.         

  122.         if initplay and initdata:

  123.             break

  124. 

  125.     return ccenabled, creditdata, recvids, recchans, recmixes, recplayl

  126. 

  127. if __name__ == "__main__":

  128.     from sys import argv

  129.     vidl = argv

  130.     vidl.pop(0)

  131.     for video in vidl:

  132.         print(getmetadata(video))