From fea0c31d0828853917d2b6101769be488e329078 Mon Sep 17 00:00:00 2001 From: elburg Date: Sun, 5 Feb 2023 12:07:35 -0800 Subject: [PATCH 1/2] oops forgot to readd stuff cuz i moved it >~< --- category/deviantart.py | 17 ++++++ category/furaffinity.py | 9 +++ main.py | 2 +- misc/getpub.py | 7 +++ standards/exif.py | 81 +++++++++++++++++++++++++ standards/iptc.py | 0 standards/matroska.py | 0 standards/png.py | 0 standards/xmp.py | 127 ++++++++++++++++++++++++++++++++++++++++ 9 files changed, 242 insertions(+), 1 deletion(-) create mode 100644 category/deviantart.py create mode 100644 category/furaffinity.py create mode 100644 misc/getpub.py create mode 100644 standards/exif.py create mode 100644 standards/iptc.py create mode 100644 standards/matroska.py create mode 100644 standards/png.py create mode 100644 standards/xmp.py diff --git a/category/deviantart.py b/category/deviantart.py new file mode 100644 index 0000000..9aac679 --- /dev/null +++ b/category/deviantart.py @@ -0,0 +1,17 @@ +def makeKeyWordList(tags): + catPaths = { + 'digitalart/paintings/other': ['Digital Art', 'Paintings'] + } + output = [ + tags.da_category, + tags.license, + tags.author.username + ] + + if tags.is_mature: + output.append('is_mature') + + + for item in .get(tags.category_path,[]): + output.append(item) + \ No newline at end of file diff --git a/category/furaffinity.py b/category/furaffinity.py new file mode 100644 index 0000000..b01bf98 --- /dev/null +++ b/category/furaffinity.py @@ -0,0 +1,9 @@ +def makeKeyWordList(tags): + return [ + tags.theme, + tags.rating, + tags.species, + tags.gender, + tags.artist_url, + tags.fa_category + ] \ No newline at end of file diff --git a/main.py b/main.py index 1668ade..60b87bf 100644 --- a/main.py +++ b/main.py @@ -16,7 +16,7 @@ with open(f"{source}.json") as file: # TODO: insert method to check url or dict logging.debug("determining file type...") -import exif +from standards import exif if infojson.extension in exif.allowed_exts: logging.debug(f"exif module supports {infojson.extesnion} files. using it... ") \ No newline at end of file diff --git a/misc/getpub.py b/misc/getpub.py new file mode 100644 index 0000000..5d13a19 --- /dev/null +++ b/misc/getpub.py @@ -0,0 +1,7 @@ +def getPub(input): + if fa_category in input or input.category == 'furaffinity': + return 'Furaffinity' + elif input.category == 'deviantart': + return 'DeviantArt' + else: + return 'Unknown' \ No newline at end of file diff --git a/standards/exif.py b/standards/exif.py new file mode 100644 index 0000000..e4785a2 --- /dev/null +++ b/standards/exif.py @@ -0,0 +1,81 @@ +import logging +logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) + +allowed_exts=[ + "tif","tiff", # TIFF + "jpeg","jpg","pjpeg","pjpg","jpe", # JPEG + "webp", # Webp +# "wav","wave", # RIFF Wave + "png","apng" # PNG (since 1.2) +] + +def backupMetadata(file): + from PIL import Image, ExifTags + + logging.debug(f"loading file {file}") + img = Image.open(file) + logging.debug("file loaded. getting exif") + img_exif = img.getexif() + + full=[] + + if img_exif is None: + logging.warning("Could not load EXIF Data... Either something happened with Pillow, the file disappeared, or you don't have access to the file. Try running verbose!") + break + else: + logging.debug("loaded tags") + for key, val in img_exif.items(): + logging.debug(f"for key:'{key}', val:'{val}' in img_exif.items()") + if key in ExifTags.TAGS: + logging.debug(f"{key} is known exif tag! ({ExifTags.TAGS[key]})") + full.append({ + 'tagName': str(key), + 'realName': str(ExifTags.TAGS[key]), + 'value': str(val) + }) + else: + logging.warning(f"{key} is an unknown exif tag!") + full.append({ + 'tagName': str(key), + 'realName': None, + 'value': str(val) + }) + + logging.info("writing backup data") + import json + + #import tempfile + #temp = tempfile.TemporaryFile(mode='w+') + #json.dump(full,temp,sort_keys=True) + with open(f"{file}.exif_backup.json.xz",'w+b') as bfile + logging.debug(f"writing to {bfile}") + import lzma + bfile.write(lzma.compress( + bytes(json.dumps(full,sort_keys=True)), + format=FORMAT_XZ, + check=CHECK_SHA256, + preset=9 + )) + logging.debug("wrote backup data") + + logging.info("backup done!") + +def writeTags(file,tags): + from PIL import Image + image=Image.open(file) + exif = image.getexif() + + if artist in tags: + if type(tag.artist) == str: + exif[0x013b] = tags.artist + elif tags.category == "deviantart": + exif[0x013b] = tags.artist.username + + if title in tags: + exif[0x0010e] = tags.title + + if description in tags: + exif[0x9286] = tags.description + + + \ No newline at end of file diff --git a/standards/iptc.py b/standards/iptc.py new file mode 100644 index 0000000..e69de29 diff --git a/standards/matroska.py b/standards/matroska.py new file mode 100644 index 0000000..e69de29 diff --git a/standards/png.py b/standards/png.py new file mode 100644 index 0000000..e69de29 diff --git a/standards/xmp.py b/standards/xmp.py new file mode 100644 index 0000000..ca6f3aa --- /dev/null +++ b/standards/xmp.py @@ -0,0 +1,127 @@ +"XMP Manipulator for gallery-dl" +import logging +logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) + +allowed_exts=[ + "html", "htm", "xhtml", "xht", # HTML + "tif", "tiff", # TIFF + "jpeg", "jpg", "pjpeg", "pjpg", "jpe", # JPEG + "jp2", "jpf", "jpx", "jpm", "mj2", # JPEG 2000 + "png", "apng", # PNG + "gif", # GIF + "mp3", # MPEG-1 / MPEG-2 Layer III + "mp4", "m4a", "m4p", "m4b", "m4r", "m4v", # MPEG-4 + "mov", "qt", # Apple QuickTime .mov + "pdf", # PDF + "webp" # WebP +] + +# def backupMetadata(file: str, outputFileExt: str = "xmp_backup.json.xz") -> bool: +# """Exports XMP metadata inside of a file. + +# Parameters +# ---------- +# file : str +# Location of the file relative to the working directory. +# outputFileExt : str, optional +# The suffix used for the backup file. By default, the output file will be formatted as such: +# .xmp_backup.json.xz + +# Returns +# ------- +# bool +# True, if operation was successful. +# False, if operation failed. +# """ +# from libxmp.utils import file_to_dict + +# import json +# with open(f"{file}.xmp_backup.json.xz",'w+b') as bfile +# logging.debug(f"writing to {bfile}") +# import lzma +# bfile.write(lzma.compress( +# bytes(json.dumps(file_to_dict(file),sort_keys=True)), +# format=FORMAT_XZ, +# check=CHECK_SHA256, +# preset=9 +# )) +# logging.debug("wrote backup data") + +# logging.info("backup done!") + +def backupMetadata(file): + import pyexiv2 + data = pyexiv2.ImageMetadata(file) + data.read() + + full = [] + + for key in data.xmp_keys: + full.append([ + key, + data[key].raw_value + ]) + + with open(f"{file}.xmp_backup.json.xz",'w+b') as bfile + import lzma, json + bfile.write(lzma.compress( + bytes(json.dumps(full)), + format=FORMAT_XZ, + check=CHECK_SHA256, + preset=9 + )) + +# def writeTags(file: str, tags: dict) -> bool: +# """Writes XMP tags to file taking the info.json file as a dict. + + +# """ +# from libxmp import XMPFiles, consts +# xmpfile = XMPFiles(file_path=file,) +# if artist in tags: +# if type(tags.artist) == str: +# xmp.set_property(consts.XMP_NS_DC, u'creator', tags.artist.decode('unicode-escape')) + +def writeTags(file, tags): + import pyexiv2 + xmptags = pyexiv2.ImageMetadata(file) + xmptags.read() + + if artist in tags: + if type(tag.artist) == str: + xmptags['Xmp.dc.creator'] = tags.artist + elif tags.category == "deviantart": + xmptags['Xmp.dc.creator'] = tags.artist.username + + if title in tags: + xmptags['Xmp.dc.title'] = tags.title + + if description in tags: + xmptags['Xmp.dc.description'] = tags.description + + if date in tags: + xmptags['Xmp.dc.date'] = tags.date + + from ..misc.getpub import getPub + xmptags['Xmp.dc.publisher'] = getPub(tags) + del getPub + + keywords=[] + + keywords.append(tags.category) + + if tags in tags: + for item in tags.tags: + if item not in keywords: keywords.append(item) + + if tags.category == 'deviantart': + from ..category.deviantart import makeKeyWordList + for item in makeKeyWordList(tags): + if item not in keywords: keywords.append(item) + elif tags.category == 'furaffinity': + from ..category.furaffinity import makeKeyWordList + for item in makeKeyWordList(tags): + if item not in keywords: keywords.append(item) + + + \ No newline at end of file From e15437252e684c4baba3bf28f78c898f97cb5f6f Mon Sep 17 00:00:00 2001 From: elburg Date: Sun, 5 Feb 2023 12:09:44 -0800 Subject: [PATCH 2/2] kyaaa!! >~< --- exif.py | 81 ---------------------------------- getpub.py | 7 --- iptc.py | 0 main.py | 22 ---------- matroska.py | 0 png.py | 0 xmp.py | 123 ---------------------------------------------------- 7 files changed, 233 deletions(-) delete mode 100644 exif.py delete mode 100644 getpub.py delete mode 100644 iptc.py delete mode 100644 main.py delete mode 100644 matroska.py delete mode 100644 png.py delete mode 100644 xmp.py diff --git a/exif.py b/exif.py deleted file mode 100644 index e4785a2..0000000 --- a/exif.py +++ /dev/null @@ -1,81 +0,0 @@ -import logging -logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) - -allowed_exts=[ - "tif","tiff", # TIFF - "jpeg","jpg","pjpeg","pjpg","jpe", # JPEG - "webp", # Webp -# "wav","wave", # RIFF Wave - "png","apng" # PNG (since 1.2) -] - -def backupMetadata(file): - from PIL import Image, ExifTags - - logging.debug(f"loading file {file}") - img = Image.open(file) - logging.debug("file loaded. getting exif") - img_exif = img.getexif() - - full=[] - - if img_exif is None: - logging.warning("Could not load EXIF Data... Either something happened with Pillow, the file disappeared, or you don't have access to the file. Try running verbose!") - break - else: - logging.debug("loaded tags") - for key, val in img_exif.items(): - logging.debug(f"for key:'{key}', val:'{val}' in img_exif.items()") - if key in ExifTags.TAGS: - logging.debug(f"{key} is known exif tag! ({ExifTags.TAGS[key]})") - full.append({ - 'tagName': str(key), - 'realName': str(ExifTags.TAGS[key]), - 'value': str(val) - }) - else: - logging.warning(f"{key} is an unknown exif tag!") - full.append({ - 'tagName': str(key), - 'realName': None, - 'value': str(val) - }) - - logging.info("writing backup data") - import json - - #import tempfile - #temp = tempfile.TemporaryFile(mode='w+') - #json.dump(full,temp,sort_keys=True) - with open(f"{file}.exif_backup.json.xz",'w+b') as bfile - logging.debug(f"writing to {bfile}") - import lzma - bfile.write(lzma.compress( - bytes(json.dumps(full,sort_keys=True)), - format=FORMAT_XZ, - check=CHECK_SHA256, - preset=9 - )) - logging.debug("wrote backup data") - - logging.info("backup done!") - -def writeTags(file,tags): - from PIL import Image - image=Image.open(file) - exif = image.getexif() - - if artist in tags: - if type(tag.artist) == str: - exif[0x013b] = tags.artist - elif tags.category == "deviantart": - exif[0x013b] = tags.artist.username - - if title in tags: - exif[0x0010e] = tags.title - - if description in tags: - exif[0x9286] = tags.description - - - \ No newline at end of file diff --git a/getpub.py b/getpub.py deleted file mode 100644 index 5d13a19..0000000 --- a/getpub.py +++ /dev/null @@ -1,7 +0,0 @@ -def getPub(input): - if fa_category in input or input.category == 'furaffinity': - return 'Furaffinity' - elif input.category == 'deviantart': - return 'DeviantArt' - else: - return 'Unknown' \ No newline at end of file diff --git a/iptc.py b/iptc.py deleted file mode 100644 index e69de29..0000000 diff --git a/main.py b/main.py deleted file mode 100644 index 60b87bf..0000000 --- a/main.py +++ /dev/null @@ -1,22 +0,0 @@ -# module imports -import sys -import json -import logging - -# init logging -logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) - -# variable init -infojson={} -source=sys.argv[1] - -logging.info(f"loading info.json for {source}...") -with open(f"{source}.json") as file: - infojson=json.load(file) - -# TODO: insert method to check url or dict -logging.debug("determining file type...") -from standards import exif -if infojson.extension in exif.allowed_exts: - logging.debug(f"exif module supports {infojson.extesnion} files. using it... ") - \ No newline at end of file diff --git a/matroska.py b/matroska.py deleted file mode 100644 index e69de29..0000000 diff --git a/png.py b/png.py deleted file mode 100644 index e69de29..0000000 diff --git a/xmp.py b/xmp.py deleted file mode 100644 index ba9cb18..0000000 --- a/xmp.py +++ /dev/null @@ -1,123 +0,0 @@ -"XMP Manipulator for gallery-dl" -import logging -logging.basicConfig(stream=sys.stderr, level=logging.DEBUG) - -allowed_exts=[ - "html", "htm", "xhtml", "xht", # HTML - "tif", "tiff", # TIFF - "jpeg", "jpg", "pjpeg", "pjpg", "jpe", # JPEG - "jp2", "jpf", "jpx", "jpm", "mj2", # JPEG 2000 - "png", "apng", # PNG - "gif", # GIF - "mp3", # MPEG-1 / MPEG-2 Layer III - "mp4", "m4a", "m4p", "m4b", "m4r", "m4v", # MPEG-4 - "mov", "qt", # Apple QuickTime .mov - "pdf", # PDF - "webp" # WebP -] - -# def backupMetadata(file: str, outputFileExt: str = "xmp_backup.json.xz") -> bool: -# """Exports XMP metadata inside of a file. - -# Parameters -# ---------- -# file : str -# Location of the file relative to the working directory. -# outputFileExt : str, optional -# The suffix used for the backup file. By default, the output file will be formatted as such: -# .xmp_backup.json.xz - -# Returns -# ------- -# bool -# True, if operation was successful. -# False, if operation failed. -# """ -# from libxmp.utils import file_to_dict - -# import json -# with open(f"{file}.xmp_backup.json.xz",'w+b') as bfile -# logging.debug(f"writing to {bfile}") -# import lzma -# bfile.write(lzma.compress( -# bytes(json.dumps(file_to_dict(file),sort_keys=True)), -# format=FORMAT_XZ, -# check=CHECK_SHA256, -# preset=9 -# )) -# logging.debug("wrote backup data") - -# logging.info("backup done!") - -def backupMetadata(file): - import pyexiv2 - data = pyexiv2.ImageMetadata(file) - data.read() - - full = [] - - for key in data.xmp_keys: - full.append([ - key, - data[key].raw_value - ]) - - with open(f"{file}.xmp_backup.json.xz",'w+b') as bfile - import lzma, json - bfile.write(lzma.compress( - bytes(json.dumps(full)), - format=FORMAT_XZ, - check=CHECK_SHA256, - preset=9 - )) - -# def writeTags(file: str, tags: dict) -> bool: -# """Writes XMP tags to file taking the info.json file as a dict. - - -# """ -# from libxmp import XMPFiles, consts -# xmpfile = XMPFiles(file_path=file,) -# if artist in tags: -# if type(tags.artist) == str: -# xmp.set_property(consts.XMP_NS_DC, u'creator', tags.artist.decode('unicode-escape')) - -def writeTags(file, tags): - import pyexiv2 - xmptags = pyexiv2.ImageMetadata(file) - xmptags.read() - - if artist in tags: - if type(tag.artist) == str: - xmptags['Xmp.dc.creator'] = tags.artist - elif tags.category == "deviantart": - xmptags['Xmp.dc.creator'] = tags.artist.username - - if title in tags: - xmptags['Xmp.dc.title'] = tags.title - - if description in tags: - xmptags['Xmp.dc.description'] = tags.description - - if date in tags: - xmptags['Xmp.dc.date'] = tags.date - - from getpub import getPub - xmptags['Xmp.dc.publisher'] = getPub(tags) - del getPub - - keywords=[] - - if tags in tags: for item in tags.tags: if item not in keywords: keywords.append(item) - - if theme in tags and tags.theme not in keywords: - keywords.append(tags.theme) - - if rating in tags and tags.rating not in keywords: - keywords.append(tags.rating) - - if species in tags and tags.species not in keywords: - keywords.append(tags.species) - - if gender in tags and tags.gender not in keywords: - keywords \ No newline at end of file