commit d1f19336c2c8d920af827ac3e78138b991837075 Author: JoshEatsAll Date: Thu Jul 4 16:47:53 2024 +0200 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2e9768f --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.db +./**/__pycache__ +**.pyc +.cache +*replit* +.config +venv +.upm \ No newline at end of file diff --git a/data/locale/en_US.err.yaml b/data/locale/en_US.err.yaml new file mode 100644 index 0000000..246448e --- /dev/null +++ b/data/locale/en_US.err.yaml @@ -0,0 +1,6 @@ +err: + gen: + UnknownFeedType: "Unknown Feed" + UnknownFeedDescription: "Unknown Feed Description" + UnknownFeedName: "Unknown Feed Name" + \ No newline at end of file diff --git a/data/locale/en_US.gen.feed.yaml b/data/locale/en_US.gen.feed.yaml new file mode 100644 index 0000000..da33517 --- /dev/null +++ b/data/locale/en_US.gen.feed.yaml @@ -0,0 +1,10 @@ +gen: + feed: + any: + title: + search: "Search results for %s" + description: + search: + limited: "The top %d results for the search query '%s'" + unlimted: "All search results for the search query '%s'\n\n(WARNING: might be unstable!!!)" + \ No newline at end of file diff --git a/data/templates/errorpage.404.basic.xml.jinja b/data/templates/errorpage.404.basic.xml.jinja new file mode 100644 index 0000000..4e40a8c --- /dev/null +++ b/data/templates/errorpage.404.basic.xml.jinja @@ -0,0 +1,13 @@ + + + + {% if result.errors %} + + {% for item in result.errors %} + {{ result.errors[item].remark }} + {% endfor %} + + {% endif %} + + {{ outbody.remark }} + \ No newline at end of file diff --git a/data/templates/feed.rss2.xml.jinja b/data/templates/feed.rss2.xml.jinja new file mode 100644 index 0000000..c8a0df0 --- /dev/null +++ b/data/templates/feed.rss2.xml.jinja @@ -0,0 +1,47 @@ +{# Metadata generator #} + + + + {% if result.rss.title %} + {% if ask.rss.type == "search" %} + {{ strings.gen.feed.any.title.search.format(result.rss.title) }} + {% elif ask.rss.type == "status" %} + {{ strings.gen.feed.any.title.status }} + {% else %} + {{ strings.err.gen.UnknownFeedType }} + {% endif %} + {% else %} + {{ strings.err.gen.UnknownFeedName }} + {% endif %} + {% if result.rss.description %} + {% if ask.rss.type == "search" %} + {% if ask.query.limit not 0 %} + {{ strings.gen.feed.any.description.search.limited.format(ask.query.limit,result.rss.title) }} + {% else %} + {{ strings.gen.feed.any.description.search.unlimited.format(result.rss.title) }} + {% endif %} + {% elif ask.rss.type == "status" %} + {{ strings.gen.feed.any.description.status }} + + {% else %} + {{ strings.err.gen.UnknownFeedDescription }} + {% endif %} + {% endif %} + {% if result.rss.link %} + {% if ask.source %} + {{ ask.source.urls.goto % result.rss.link }} + {% else %} + {{ result.rss.link }} + {% endif %} + {% endif %} + {{ ask.requestDate }} + + {{ "%-%".format(ask.lang.lower(),ask.region.lower()) }} + superTinySearch API Frontend v{{ meta.api.version }} using Python Jinja v3 + + {% for item in result.rss.items %} + {% include 'xml.blocks.rss2.item.xml.jinja' %} + {% endfor %} + + + \ No newline at end of file diff --git a/data/templates/hidden.commServ.submitURL.ok.html.jinja b/data/templates/hidden.commServ.submitURL.ok.html.jinja new file mode 100644 index 0000000..202bd7d --- /dev/null +++ b/data/templates/hidden.commServ.submitURL.ok.html.jinja @@ -0,0 +1,29 @@ + + + + submitted url! + + +

url submitted!

+

thank you for your submission! [sts] will process your contributions and will add or remove entires based on your input.



+ to recap, you submitted...
+ {% for action in input.actions %} + {% if action.what == "scanurl" %} +

a URL to scan...

+

we will scan the page or file at {{ action.where }} and add that and any other URLs found from here...



+ {% elif action.what == "removeurl" %} +

a URL to remove from our databases...

+ {% if action.why == "personal" %} +

after we confirm that {{ action.where }} contains very personal information, we will remove this and blacklist our scanners to prevent this from being scanned ever again...

+ {% elif action.why == "malicious" %} +

after we confirm that {{ action.where }} contains malicious content, we will remove this and blacklist our scanners to prevent this from being scanned ever again...

+

(if this url is exceptionally dangerous, please report this url to it's hosting provider, or to the proper authorities.)

+ {% elif action.why == "false" %} +

after we confirm that {{ action.where }} contains false content, we will remove this and blacklist our scanners to prevent this from being scanned ever again...

+ {% endif %} + {% endif %} + {% endfor %} +
+ thaks for contributing to a foss project! + + \ No newline at end of file diff --git a/data/templates/xml.blocks.rss2.item.xml.jinja b/data/templates/xml.blocks.rss2.item.xml.jinja new file mode 100644 index 0000000..8b4dec3 --- /dev/null +++ b/data/templates/xml.blocks.rss2.item.xml.jinja @@ -0,0 +1,14 @@ + + {{ item.title }} + {{ item.link }} + {{ item.description|e }} + {% if item.author != "None" %} + {{ item.author }} + {% endif %} + {% if item.tags %} + {{ item.tags|join('/') }} + {% endif %} + {% if item.indexDate != "None" %} + {{ item.indexDate }} + {% endif %} + \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..212a856 --- /dev/null +++ b/main.py @@ -0,0 +1,76 @@ +exit() + +__version__ = "0.0" + +from flask import Flask as wserv + +app = wserv('superTinySearch API Runtime') + +from flask import render_template + +import sys + +debugmode = False +if "debug" in sys.argv: + debugmode = True +del sys + + +def makeServerHeader(): + import sys + return f"superTinySearch API/{__version__} (Unix) (Python v{sys.version})" + + +# @app.errorhandler(404) +# def not_found(error): +# from flask import request as rq +# if rq.args.get('returnFormat','') == "xml" + + +@app.route("/") +def noHumans(): + from flask import Response + resp = Response(response="running", status=200, mimetype="text/plain") + resp.headers["Server"] = makeServerHeader + resp.headers["Accept"] = "text/plain" + return resp + + +@app.route("/about") +def aboutThis(): + from flask import Response + resp = Response(response=f"superTinySearch API v{__version__}", + status=200, + mimetype="text/plain") + resp.headers["Server"] = makeServerHeader + resp.headers["Accept"] = "text/plain" + return resp + + +@app.route("/apifront?returnFormat=&srcHostname=", + methods=['GET', 'POST']) +def apiRun(rf='xml', src=None): + from flask import request as rq + if rq.method == 'GET': + return 'GET REQUESTS NOT ALLOWED' + else: + from src import frontend + out = frontend.parsePOST(rq.form.to_dict(flat=False), rf, src) + del frontend + from flask import Response + resp = Response(response=out.content, status=500, mime="text/plain") + if rf == "xml": + resp.headers["Content-Type"] = "text/xml; charset=utf-8" + elif rf == "json": + resp.headers["Content-Type"] = "application/json; charset=utf-8" + else: + resp.headers["Content-Type"] = "text/plain; charset=utf-8" + resp.headers["Server"] = makeServerHeader() + resp.status = 200 + return resp + + +if __name__ == '__main__': + app.run(host="0.0.0.0", port=443) +else: + raise "INVALID USAGE OF API APP" diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..11a494d --- /dev/null +++ b/poetry.lock @@ -0,0 +1,134 @@ +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" + +[[package]] +name = "flask" +version = "2.2.2" +description = "A simple framework for building complex web applications." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +click = ">=8.0" +importlib-metadata = {version = ">=3.6.0", markers = "python_version < \"3.10\""} +itsdangerous = ">=2.0" +Jinja2 = ">=3.0" +Werkzeug = ">=2.2.2" + +[package.extras] +async = ["asgiref (>=3.2)"] +dotenv = ["python-dotenv"] + +[[package]] +name = "importlib-metadata" +version = "6.0.0" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"] +perf = ["ipython"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8", "importlib-resources (>=1.3)"] + +[[package]] +name = "itsdangerous" +version = "2.1.2" +description = "Safely pass data to untrusted environments and back." +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "markupsafe" +version = "2.1.2" +description = "Safely add untrusted strings to HTML/XML markup." +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "numpy" +version = "1.22.2" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = false +python-versions = ">=3.8" + +[[package]] +name = "werkzeug" +version = "2.2.2" +description = "The comprehensive WSGI web application library." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +MarkupSafe = ">=2.1.1" + +[package.extras] +watchdog = ["watchdog"] + +[[package]] +name = "zipp" +version = "3.12.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "jaraco.functools", "more-itertools", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"] + +[metadata] +lock-version = "1.1" +python-versions = ">=3.8.0,<3.9" +content-hash = "400928830379454435838ae9f145feaafae43a4a0c4f65edbe32de50c7a3aa9c" + +[metadata.files] +click = [] +colorama = [] +flask = [] +importlib-metadata = [] +itsdangerous = [] +jinja2 = [] +markupsafe = [] +numpy = [] +werkzeug = [] +zipp = [] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..51b0e41 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[tool.poetry] +name = "python-template" +version = "0.1.0" +description = "" +authors = ["Your Name "] + +[tool.poetry.dependencies] +python = ">=3.8.0,<3.9" +numpy = "^1.22.2" +Flask = "^2.2.2" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" \ No newline at end of file diff --git a/src/database/searchDB.py b/src/database/searchDB.py new file mode 100644 index 0000000..7413a94 --- /dev/null +++ b/src/database/searchDB.py @@ -0,0 +1,66 @@ +class Accessor: + _DBFILE = "~/sts-api/.db/search.sqlite3" + + def _refresh(self): + if self.buffers.db == None: + import sqlite3 + self.buffers.db = sqlite3.connect(self._DBFILE) + self.buffers.dbcursor = self.buffers.db.cursor() + + def list(self, type="*"): + self._refresh() + self.lastOut = self.buffers.dbcursor.execute( + "SELECT {} FROM index".format(type)) + return self.lastOut + + def search(self, query, limit=None, offset=None): + self._refresh() + if self.mode == "ro" or self.mode == "rw": + import re + if query != None: + self.search.query = re.compile(f"^*{query}*$", re.IGNORECASE) + self.search.limit = limit + self.search.offset = offset + if limit == None: + self.lastOut = re.search(self.search.query, str(self.list())) + return self.lastOut + else: + self.lastOut = re.search(self.search.query, str(self.list())) + return self.lastOut[:25] + + def addEntry(self, entryData): + if self.mode == "ro": + raise ImportError + self._refresh() + if entryData.url in self.list("url"): + raise "EntryExists" + self.buffers.dbcursor.execute(f""" + INSERT INTO index VALUES + ('{entryData.fullURL}', '{entryData.faviconURL}', '{entryData.protocol}', '{entryData.title}', '{entryData.description}') + """) + self.buffers.db.commit() + return True + def removeEntry(self, filters): + if self.mode == "ro": + raise ImportError + self._refresh() + query="DELETE FROM index WHERE " + if filters.full: + query+=f"fullURL = {filters.fullURL}" + elif filters.hostname: + query+=f"fullURL LIKE %{filters.hostname}%" + + query+=";" + self.buffers.dbcursor.execute(query) + self.buffers.db.commit() + return True + def __init__(self, mode="ro"): + self.mode = mode + self.search = {'query': None, 'limit': 20, 'offset': 0} + self.lastOut = None + self.buffers = { + 'db': None, + 'dbcursor': None, + 'read': None, + 'write': None + } diff --git a/src/frontend.py b/src/frontend.py new file mode 100644 index 0000000..d603d73 --- /dev/null +++ b/src/frontend.py @@ -0,0 +1,9 @@ +def parsePOST(rqform,src): + if rqform.action=="searchQuery": + from database import searchDB + results=searchDB.Accessor() + results.search(query=rqform.query, limit=rqform.limit, offset=rqform.offset) + return results.scannerResults + elif rqform.action=="genFeed": + if rqform.subAction=="operationStatus": + \ No newline at end of file