from tagem import TagemDB

from collections import Counter

import json

import magic

import base64

import os

import mimetype_utils

import re

mimetype_utils.load_cached_mimetypes("cached_mimetypes.json")

subtitle_langs:tuple = ("en","de")

FAKETAG_hassubtitles:int = 1234567890

tagemdb = TagemDB(user_id=4)

def get_files_directly_tagged(tagid:int):

return [x[0] for x in tagemdb.qry(f"SELECT file FROM file2tag WHERE tag={tagid}")]

def get_thumb_fp(fileid:int, tagids:list):

return fp

tagem_fileids:list = list(set(get_files_directly_tagged(237870)+get_files_directly_tagged(65574)+get_files_directly_tagged(239895)+get_files_directly_tagged(239896)+get_files_directly_tagged(239897)+get_files_directly_tagged(239898)+get_files_directly_tagged(239900)+get_files_directly_tagged(116236)+get_files_directly_tagged(3020)+get_files_directly_tagged(6810)+get_files_directly_tagged(239922)+get_files_directly_tagged(236111)+get_files_directly_tagged(239901)+get_files_directly_tagged(239902)+get_files_directly_tagged(239926)+get_files_directly_tagged(239923)+get_files_directly_tagged(240024)+[2200548,4900172,2199768,1985,2203575,2199933,4829561]))

tagem_fileid2skipfirstseconds:dict = {

"that fallout video intro":16.108312,

}

introduction_to_each_category:dict = {

}

tagem_tagids:dict = { # NOTE: As of Python 3.7, regular dicts are guaranteed to be ordered

}

remove_a_if_already_tagged_b:tuple = (

(743,239900), # Traditional (Music,Song)

(744,1987), # Sad (Music,Song)

)

tagem_tagids__ordered_keys:list = [key for key in tagem_tagids]

tagem_tagids__as_indices:list = [tagem_tagids[key] for key in tagem_tagids__ordered_keys]

tagem_tagids__ordered_keys__including_hidden_tagids:list = tagem_tagids__ordered_keys+[

217409, # French Socialist Song

]

total_filesize:int = 0

filesizes:list = []

all_filepaths_added_to_server:list = []

def fp_of_file_added_to_server(fileid:int, fp:str):

all_filepaths_added_to_server.append(fp)

def set_symlink(linkat:str, linkto:str):

os.symlink(linkto, linkat)

all_tags_associated_with_chosen_files:list = []

fileid2associatedtags:list = []

i:int = 0

for fp in (

,

):

set_symlink(f"/home/vangelic/repos/compsky/static-and-chat-server/files/large/{i:04d}", fp)

fp_of_file_added_to_server(None, fp)

i += 1

for fp in (

, # "files/.../path/to/some.html",

):

final_result:list = []

is_modified:bool = False

with open(fp,"r") as f:

for orig_line in f.read().split("\n"):

line:str = orig_line

m = re.search(''' data-src="(/media/vangelic/DATA/[^"]+)"''', line)

if m is not None:

_fp:str = m.group(1)

set_symlink(f"files/large/{i:04d}", _fp)

fp_of_file_added_to_server(None, _fp)

if ' src="' in line:

line = re.sub(' src="[^"]*"', f' src="../large/{i:04d}"', line)

else:

line = re.sub(' data-src="', f' src="../large/{i:04d}" data-src="', line)

i += 1

final_result.append(line)

if line != orig_line:

is_modified = True

if final_result[-1] == "":

final_result.pop()

if is_modified:

with open(fp,"w") as f:

for line in final_result:

f.write(line+"\n")

captcha_video_fps:list = []

captcha_questions:list = []

process_captcha_clips_and_questions(i, captcha_video_fps, captcha_questions)

for fp in captcha_video_fps:

set_symlink(f"/home/vangelic/repos/compsky/static-and-chat-server/files/large/{i:04d}", fp)

fp_of_file_added_to_server(None, fp)

i += 1

tagem_fileid2indx:dict = {}

fileid2alternative_thumbnail:dict = {}

def fileid2alternative_thumbnail__process(dirpath:str):

fileid2alternative_thumbnail[int(m.group(1))] = fp

fileid2alternative_thumbnail__process("/media/vangelic/DATA/static-server-files/audio-thumbs")

print(f"Registered {len(fileid2alternative_thumbnail)} alternative thumbnails")

tag2thumbnail:dict = {

}

for key,val in tag2thumbnail.items():

if type(val) is tuple:

tag2thumbnail[key] = val[1]

else:

mimetype:str = "image/jpeg" if val.endswith(".jpg") else ("image/webp" if val.endswith(".webp") else "image/png")

with open(f"/media/vangelic/DATA/static-server-files/tag-thumbs{val}","rb") as f:

tag2thumbnail[key] = "data:"+mimetype+";base64,"+base64.encodebytes(f.read()).replace(b"\n",b"").decode()

fileid2subtitles:dict = {}

def get_subtitle_vtt_for_fileid(fp:str, lang:str, fileid:int):

with open(fp,"rb") as f:

subtitle:list = [subtitle_langs.index(lang),"data:text/vtt;base64,"+base64.encodebytes(process_vtt_contents(f.read())).replace(b"\n",b"").decode()]

return subtitle

def process_vtt_contents(s:str):

return b"\n".join(lines)

for fname in os.listdir("/media/vangelic/DATA/static-server-files/audio-subtitles"):

if fname.startswith("test."):

continue

fp:str = f"/media/vangelic/DATA/static-server-files/audio-subtitles/{fname}"

m = re.search("^([0-9]+)[.](.*)$", fname)

if m is None:

continue

fileid:int = int(m.group(1))

lang:str = m.group(2)

if lang not in subtitle_langs:

print(f"WARNING: Skipping unrecognised subtitle language: {lang}")

continue

subtitle:str = get_subtitle_vtt_for_fileid(fp,lang,fileid)

if fileid in fileid2subtitles:

fileid2subtitles[fileid].append(subtitle)

else:

fileid2subtitles[fileid] = [subtitle]

for fileid in tagem_fileids:

if fileid == 7155: # Endless Space 2 hour compilation

continue

fp:str = None

if os.path.exists(f"/media/vangelic/DATA/static-server-files/compressed_videos/by-tagem-fileid/{fileid}"):

fp = f"/media/vangelic/DATA/static-server-files/compressed_videos/by-tagem-fileid/{fileid}"

if fp is None:

fp = tagemdb.qry("SELECT CONCAT(d.full_path,f.name) FROM file f JOIN dir d ON d.id=f.dir WHERE f.id="+str(fileid))[0][0]

if type(fp) is bytes:

fp = fp.decode()

if not os.path.isfile(fp):

fp = None

if fp is None:

for (_fp,) in tagemdb.qry("SELECT CONCAT(d.full_path,f.name) FROM file_backup f JOIN dir d ON d.id=f.dir WHERE f.file="+str(fileid)):

if os.path.isfile(_fp):

fp = _fp.decode()

break

if fp is None:

print(f"Can't find file for id {fileid}")

continue

tagids:list = [x[0] for x in tagemdb.qry(f"SELECT t2pt.parent FROM file2tag f2t JOIN tag2parent_tree t2pt ON t2pt.id=f2t.tag WHERE f2t.file={fileid}")]

# tagids = [116236 if (x==229017) else x for x in tagids] # Include Fashwave in Retrowave

if fileid == 4902000:

tagids = [239926]

if fileid == 7678:

tagids.append(1219)

if fileid == 4900220: # Field of poppies piano

tagids.append(235314) # Anti-War Song

if fileid == 6722: # Dear White People

tagids = [239898] # weird songs

all_tags_associated_with_chosen_files += tagids

thumb_fp:str = get_thumb_fp(fileid, tagids)

thumb_str:str = 0

if fileid == 1919273: # German anthem -> deutsch

thumb_str = tagem_tagids__ordered_keys__including_hidden_tagids.index(2494)

elif fileid == 1919275: # French anthem -> fren

thumb_str = tagem_tagids__ordered_keys__including_hidden_tagids.index(2652)

elif thumb_fp is None:

for tagid in tag2thumbnail:

if fileid == 1534:

print(tagid, tagid in tagids)

if tagid == 0:

continue

if tagid in tagids:

thumb_str = tagem_tagids__ordered_keys__including_hidden_tagids.index(tagid)

break

else:

contents:bytes = None

with open(thumb_fp,"rb") as f:

contents = f.read()

mimetype:str = None

if thumb_fp.endswith(".jpg"):

mimetype = "image/jpeg"

elif thumb_fp.endswith(".webp"):

mimetype = "image/webp"

elif thumb_fp.endswith(".png"):

mimetype = "image/png"

else:

mimetype = magic.from_buffer(contents, mime=True)

if mimetype not in ("image/jpeg","image/webp","image/png"):

raise ValueError(f"ERROR: mimetype is {mimetype} for {thumb_fp}")

thumb_str = "data:"+mimetype+";base64,"+base64.encodebytes(contents).replace(b"\n",b"").decode()

file_mimetype:str = mimetype_utils.guess_mimetype(fp)

file_mimetype = mimetype_utils.standardise_mimetype(file_mimetype, fp)

filtered_tagids:list = [x for x in tagids]

for a,b in remove_a_if_already_tagged_b:

if (b in filtered_tagids) and (a in filtered_tagids):

filtered_tagids = [x for x in filtered_tagids if x!=a]

file_visible_tagids:list = list(set([tagem_tagids__ordered_keys.index(x) for x in filtered_tagids if x in tagem_tagids]))

subtitles:list = fileid2subtitles.get(fileid)

if subtitles is None:

subtitles = []

else:

file_visible_tagids.append(tagem_tagids__ordered_keys.index(FAKETAG_hassubtitles))

if len(file_visible_tagids) == 0:

tagnames:list = [x[0].decode() for x in tagemdb.qry(f"SELECT name FROM tag t JOIN file2tag f2t ON f2t.tag=t.id WHERE f2t.file={fileid}")]

print(f"WARNING: No visible tagids for file {fileid} {fp} {json.dumps(tagnames)}")

continue

tagem_fileid2indx[fileid] = i

fileid2associatedtags.append([i,thumb_str,file_visible_tagids,file_mimetype,subtitles])

set_symlink(f"/home/vangelic/repos/compsky/static-and-chat-server/files/large/{i:04d}", fp)

fp_of_file_added_to_server(fileid, fp)

i += 1

tag2thumbnail = {(0 if (key==0) else tagem_tagids__ordered_keys__including_hidden_tagids.index(key)):val for key,val in tag2thumbnail.items()}

for name,description,firstmedia,filepaths in (

,

):

tagindx:int = None

default_thumb_str:str = 0

if name in tagem_tagids__as_indices:

tagindx = tagem_tagids__as_indices.index(name)

if tagindx in tag2thumbnail:

default_thumb_str = tag2thumbnail[tagindx]

else:

tagindx = len(tagem_tagids__as_indices)

tagem_tagids__as_indices.append(name)

for fp_indx_in_set, fp in enumerate(filepaths):

thumb_str:str = default_thumb_str

if type(fp) is tuple:

if fp[0] == "PICK_ONE":

fp = fp[1][0] # TODO: Choose at random, but only when browser_cache_is_different (i.e. version v=? updated)

if type(fp) is tuple:

thumb_str = fp[1]

fp = fp[0]

if fp in all_filepaths_added_to_server:

print("WARNING: file already included:", fp)

continue

file_visible_tagids:list = [tagindx]

subtitles:list = []

fake_fileid:int = max(tagem_fileid2indx) + 1

tagem_fileid2indx[fake_fileid] = i

if os.path.isfile(fp+".vtt"):

print("Found VTT: "+fp+".vtt")

subtitles.append(get_subtitle_vtt_for_fileid(fp+".vtt", "en", fake_fileid))

file_visible_tagids.append(tagem_tagids__ordered_keys.index(FAKETAG_hassubtitles))

if firstmedia == fp_indx_in_set:

introduction_to_each_category[tagindx] = fake_fileid

file_mimetype:str = mimetype_utils.guess_mimetype(fp)

file_mimetype = mimetype_utils.standardise_mimetype(file_mimetype, fp)

fileid2associatedtags.append([i,thumb_str,file_visible_tagids,file_mimetype,subtitles])

set_symlink(f"/home/vangelic/repos/compsky/static-and-chat-server/files/large/{i:04d}", fp)

fp_of_file_added_to_server(None, fp)

i += 1

while os.path.exists(f"/home/vangelic/repos/compsky/static-and-chat-server/files/large/{i:04d}"):

os.remove(f"/home/vangelic/repos/compsky/static-and-chat-server/files/large/{i:04d}")

i += 1

all_tags_associated_with_chosen_files = [x for x in all_tags_associated_with_chosen_files if x not in tagem_tagids]

for tagid, count in Counter(all_tags_associated_with_chosen_files).most_common(50):

tagname:str = tagemdb.qry(f"SELECT name FROM tag WHERE id={tagid}")[0][0].decode()

print(f"Excluded tag associated with {count} chosen files: {tagid} {tagname}")

prev_all_filepaths_added_to_server:list = None

with open("prev_all_filepaths_added_to_server.json","r") as f:

prev_all_filepaths_added_to_server = json.load(f)

browser_cache_is_different:bool = False

for i in range(len(all_filepaths_added_to_server)):

if i == len(prev_all_filepaths_added_to_server):

break

if all_filepaths_added_to_server[i] != prev_all_filepaths_added_to_server[i]:

browser_cache_is_different = True

break

browser_cache_version:int = 0

with open("browser_cache_version.txt","r") as f:

browser_cache_version = int(f.read())

if browser_cache_is_different:

browser_cache_version += 1

with open("browser_cache_version.txt","w") as f:

f.write(str(browser_cache_version))

introduction_to_each_category = {tagem_tagids__ordered_keys.index(tagid):tagem_fileid2indx[fileid] for tagid,fileid in introduction_to_each_category.items() if (fileid in tagem_fileid2indx) and (tagid in tagem_tagids__ordered_keys)}

with open("/home/vangelic/repos/compsky/static-and-chat-server/files/static/all_.json","w") as f:

json.dump([tagem_tagids__as_indices,fileid2associatedtags,tag2thumbnail,introduction_to_each_category],f,indent=None,separators=(",",":"))

for fsz,fileid,fp in sorted(filesizes,key=lambda x:x[0],reverse=True)[:50]:

print(f"{fsz}\t{fileid}\t{fp}")

print(f"Total size of all files used by server: {total_filesize//(1024*1024)} MiB")

for fileid in fileid2alternative_thumbnail:

print(f"ERROR: Failed to use alternative thumbnail for {fileid}")

def apparmor_escstr(s:str):

return s.replace("\\","\\\\").replace("\"","\\\"").replace("[","\\[").replace("]","\\]")

if browser_cache_is_different or (len(all_filepaths_added_to_server) != len(prev_all_filepaths_added_to_server)):

with open("prev_all_filepaths_added_to_server.json.new","w") as f:

json.dump(all_filepaths_added_to_server, f)

os.rename("prev_all_filepaths_added_to_server.json.new","prev_all_filepaths_added_to_server.json")

with open("profile.largefiles.apparmor","w") as f:

for fp in all_filepaths_added_to_server:

f.write(f'"{apparmor_escstr(fp)}" r,\n')

if True:

tagem_tagnames:list = [tagem_tagids[tagid] for tagid in tagem_tagids]

def subfn(m):

return m.group(1) + str(1+tagem_tagnames.index(m.group(3))) + m.group(2)

mainpage_contents:str = None

with open("files/static/index.html","r") as f:

mainpage_contents = f.read()

mainpage_contents = re.sub('(<a href="rand.html#\\[)[0-9]+(,[^"]+\\]"><img .*alt="([^"]+)".*</img></a>)', subfn, mainpage_contents)

with open("files/static/index.html","w") as f:

f.write(mainpage_contents)

mimetype_utils.save_cached_mimetypes("cached_mimetypes.json")