commit d1f19336c2c8d920af827ac3e78138b991837075
Author: JoshEatsAll <josheatsall@redact.ed>
Date:   Thu Jul 4 16:47:53 2024 +0200

    Initial commit

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..2e9768f
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,8 @@
+.db
+./**/__pycache__
+**.pyc
+.cache
+*replit*
+.config
+venv
+.upm
\ No newline at end of file
diff --git a/data/locale/en_US.err.yaml b/data/locale/en_US.err.yaml
new file mode 100644
index 0000000..246448e
--- /dev/null
+++ b/data/locale/en_US.err.yaml
@@ -0,0 +1,6 @@
+err:
+  gen:
+    UnknownFeedType: "Unknown Feed"
+    UnknownFeedDescription: "Unknown Feed Description"
+    UnknownFeedName: "Unknown Feed Name"
+    
\ No newline at end of file
diff --git a/data/locale/en_US.gen.feed.yaml b/data/locale/en_US.gen.feed.yaml
new file mode 100644
index 0000000..da33517
--- /dev/null
+++ b/data/locale/en_US.gen.feed.yaml
@@ -0,0 +1,10 @@
+gen:
+  feed:
+    any:
+      title: 
+        search: "Search results for %s"
+      description:
+        search: 
+          limited: "The top %d results for the search query '%s'"
+          unlimted: "All search results for the search query '%s'\n\n(WARNING: might be unstable!!!)"
+        
\ No newline at end of file
diff --git a/data/templates/errorpage.404.basic.xml.jinja b/data/templates/errorpage.404.basic.xml.jinja
new file mode 100644
index 0000000..4e40a8c
--- /dev/null
+++ b/data/templates/errorpage.404.basic.xml.jinja
@@ -0,0 +1,13 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<data>
+  <result type="{{result.type}}">
+    {% if result.errors %}
+    <errors>
+      {% for item in result.errors %}
+        <error type="{{ result.errors[item].type }}" source="{{ result.errors[item].source }}">{{ result.errors[item].remark }}</error>
+      {% endfor %}
+    </errors>
+    {% endif %}
+  </result>
+  <outbody type="{{ outbody.type }}" compression="{{ outbody.compression if outbody.compression else "none" }}">{{ outbody.remark }}</outbody>
+</data>
\ No newline at end of file
diff --git a/data/templates/feed.rss2.xml.jinja b/data/templates/feed.rss2.xml.jinja
new file mode 100644
index 0000000..c8a0df0
--- /dev/null
+++ b/data/templates/feed.rss2.xml.jinja
@@ -0,0 +1,47 @@
+{# Metadata generator #}
+<?xml version='1.0' encoding='UTF-8'?>
+<rss version='2.0'>
+  <channel>
+    {% if result.rss.title %}
+      {% if ask.rss.type == "search" %}
+        <title>{{ strings.gen.feed.any.title.search.format(result.rss.title) }}</title>
+      {% elif ask.rss.type == "status" %}
+        <title>{{ strings.gen.feed.any.title.status }}</title>    
+      {% else %}
+        <title>{{ strings.err.gen.UnknownFeedType }}</title>
+      {% endif %}
+    {% else %}
+      <title>{{ strings.err.gen.UnknownFeedName }}</title>
+    {% endif %}
+    {% if result.rss.description %}
+      {% if ask.rss.type == "search" %}
+        {% if ask.query.limit not 0 %}
+          <description>{{ strings.gen.feed.any.description.search.limited.format(ask.query.limit,result.rss.title) }}</description>
+        {% else %}
+          <description>{{ strings.gen.feed.any.description.search.unlimited.format(result.rss.title) }}</description>
+        {% endif %}
+      {% elif ask.rss.type == "status" %}
+        <description>{{ strings.gen.feed.any.description.status }}</description>
+      
+      {% else %}
+        <description>{{ strings.err.gen.UnknownFeedDescription }}</description>
+      {% endif %}
+    {% endif %}
+    {% if result.rss.link %}
+      {% if ask.source %}
+      <link>{{ ask.source.urls.goto % result.rss.link }}</link>
+      {% else %}
+      <link>{{ result.rss.link }}</link>
+      {% endif %}
+    {% endif %}
+    <builddate>{{ ask.requestDate }}</builddate>
+    <pubdate></pubdate>
+    <language>{{ "%-%".format(ask.lang.lower(),ask.region.lower()) }}</language>
+    <generator>superTinySearch API Frontend v{{ meta.api.version }} using Python Jinja v3</generator>
+
+    {% for item in result.rss.items %}
+      {% include 'xml.blocks.rss2.item.xml.jinja' %}
+    {% endfor %}
+
+  </channel>
+</rss>
\ No newline at end of file
diff --git a/data/templates/hidden.commServ.submitURL.ok.html.jinja b/data/templates/hidden.commServ.submitURL.ok.html.jinja
new file mode 100644
index 0000000..202bd7d
--- /dev/null
+++ b/data/templates/hidden.commServ.submitURL.ok.html.jinja
@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>submitted url!</title>
+  </head>
+  <body>
+    <h1>url submitted!</h1>
+    <p>thank you for your submission! <i>[sts]</i> will process your contributions and will add or remove entires based on your input.</p><br><br>
+    <i>to recap, you submitted...</i><br>
+    {% for action in input.actions %}
+      {% if action.what == "scanurl" %}
+        <h2>a URL to scan...</h2>
+        <p>we will scan the page or file at <strong>{{ action.where }}</strong> and add that and any other URLs found from here...</p><br><br>
+      {% elif action.what == "removeurl" %}
+        <h2>a URL to remove from our databases...</h2>
+        {% if action.why == "personal" %}
+          <p>after we confirm that <strong>{{ action.where }}</strong> contains very personal information, we will remove this and blacklist our scanners to prevent this from being scanned ever again...</p>
+        {% elif action.why == "malicious" %}
+          <p>after we confirm that <strong>{{ action.where }}</strong> contains malicious content, we will remove this and blacklist our scanners to prevent this from being scanned ever again...</p>
+          <p>(if this url is exceptionally dangerous, please report this url to it's hosting provider, or to the proper authorities.)</p>
+        {% elif action.why == "false" %}
+           <p>after we confirm that <strong>{{ action.where }}</strong> contains false content, we will remove this and blacklist our scanners to prevent this from being scanned ever again...</p>
+        {% endif %}
+      {% endif %}
+    {% endfor %}
+    <hr>
+    <i>thaks for contributing to a <a title="free and open source software">foss project!</a></i>
+  </body>
+</html>
\ No newline at end of file
diff --git a/data/templates/xml.blocks.rss2.item.xml.jinja b/data/templates/xml.blocks.rss2.item.xml.jinja
new file mode 100644
index 0000000..8b4dec3
--- /dev/null
+++ b/data/templates/xml.blocks.rss2.item.xml.jinja
@@ -0,0 +1,14 @@
+<item>
+  <title>{{ item.title }}</title>
+  <link>{{ item.link }}</link>
+  <description>{{ item.description|e }}</description>
+  {% if item.author != "None" %}
+    <author>{{ item.author }}</author>
+  {% endif %}
+  {% if item.tags %}
+    <category>{{ item.tags|join('/') }}</category>
+  {% endif %}
+  {% if item.indexDate != "None" %}
+    <pubDate>{{ item.indexDate }}</pubDate>
+  {% endif %}
+</item>
\ No newline at end of file
diff --git a/main.py b/main.py
new file mode 100644
index 0000000..212a856
--- /dev/null
+++ b/main.py
@@ -0,0 +1,76 @@
+exit()
+
+__version__ = "0.0"
+
+from flask import Flask as wserv
+
+app = wserv('superTinySearch API Runtime')
+
+from flask import render_template
+
+import sys
+
+debugmode = False
+if "debug" in sys.argv:
+    debugmode = True
+del sys
+
+
+def makeServerHeader():
+    import sys
+    return f"superTinySearch API/{__version__} (Unix) (Python v{sys.version})"
+
+
+# @app.errorhandler(404)
+# def not_found(error):
+#   from flask import request as rq
+#     if rq.args.get('returnFormat','') == "xml"
+
+
+@app.route("/")
+def noHumans():
+    from flask import Response
+    resp = Response(response="running", status=200, mimetype="text/plain")
+    resp.headers["Server"] = makeServerHeader
+    resp.headers["Accept"] = "text/plain"
+    return resp
+
+
+@app.route("/about")
+def aboutThis():
+    from flask import Response
+    resp = Response(response=f"superTinySearch API v{__version__}",
+                    status=200,
+                    mimetype="text/plain")
+    resp.headers["Server"] = makeServerHeader
+    resp.headers["Accept"] = "text/plain"
+    return resp
+
+
+@app.route("/apifront?returnFormat=<rf>&srcHostname=<src>",
+           methods=['GET', 'POST'])
+def apiRun(rf='xml', src=None):
+    from flask import request as rq
+    if rq.method == 'GET':
+        return 'GET REQUESTS NOT ALLOWED'
+    else:
+        from src import frontend
+        out = frontend.parsePOST(rq.form.to_dict(flat=False), rf, src)
+        del frontend
+        from flask import Response
+        resp = Response(response=out.content, status=500, mime="text/plain")
+        if rf == "xml":
+            resp.headers["Content-Type"] = "text/xml; charset=utf-8"
+        elif rf == "json":
+            resp.headers["Content-Type"] = "application/json; charset=utf-8"
+        else:
+            resp.headers["Content-Type"] = "text/plain; charset=utf-8"
+        resp.headers["Server"] = makeServerHeader()
+        resp.status = 200
+        return resp
+
+
+if __name__ == '__main__':
+    app.run(host="0.0.0.0", port=443)
+else:
+    raise "INVALID USAGE OF API APP"
diff --git a/poetry.lock b/poetry.lock
new file mode 100644
index 0000000..11a494d
--- /dev/null
+++ b/poetry.lock
@@ -0,0 +1,134 @@
+[[package]]
+name = "click"
+version = "8.1.3"
+description = "Composable command line interface toolkit"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+colorama = {version = "*", markers = "platform_system == \"Windows\""}
+
+[[package]]
+name = "colorama"
+version = "0.4.6"
+description = "Cross-platform colored terminal text."
+category = "main"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+
+[[package]]
+name = "flask"
+version = "2.2.2"
+description = "A simple framework for building complex web applications."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+click = ">=8.0"
+importlib-metadata = {version = ">=3.6.0", markers = "python_version < \"3.10\""}
+itsdangerous = ">=2.0"
+Jinja2 = ">=3.0"
+Werkzeug = ">=2.2.2"
+
+[package.extras]
+async = ["asgiref (>=3.2)"]
+dotenv = ["python-dotenv"]
+
+[[package]]
+name = "importlib-metadata"
+version = "6.0.0"
+description = "Read metadata from Python packages"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+zipp = ">=0.5"
+
+[package.extras]
+docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"]
+perf = ["ipython"]
+testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8", "importlib-resources (>=1.3)"]
+
+[[package]]
+name = "itsdangerous"
+version = "2.1.2"
+description = "Safely pass data to untrusted environments and back."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "jinja2"
+version = "3.1.2"
+description = "A very fast and expressive template engine."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+MarkupSafe = ">=2.0"
+
+[package.extras]
+i18n = ["Babel (>=2.7)"]
+
+[[package]]
+name = "markupsafe"
+version = "2.1.2"
+description = "Safely add untrusted strings to HTML/XML markup."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[[package]]
+name = "numpy"
+version = "1.22.2"
+description = "NumPy is the fundamental package for array computing with Python."
+category = "main"
+optional = false
+python-versions = ">=3.8"
+
+[[package]]
+name = "werkzeug"
+version = "2.2.2"
+description = "The comprehensive WSGI web application library."
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+MarkupSafe = ">=2.1.1"
+
+[package.extras]
+watchdog = ["watchdog"]
+
+[[package]]
+name = "zipp"
+version = "3.12.0"
+description = "Backport of pathlib-compatible object wrapper for zip files"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.extras]
+docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"]
+testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "jaraco.functools", "more-itertools", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"]
+
+[metadata]
+lock-version = "1.1"
+python-versions = ">=3.8.0,<3.9"
+content-hash = "400928830379454435838ae9f145feaafae43a4a0c4f65edbe32de50c7a3aa9c"
+
+[metadata.files]
+click = []
+colorama = []
+flask = []
+importlib-metadata = []
+itsdangerous = []
+jinja2 = []
+markupsafe = []
+numpy = []
+werkzeug = []
+zipp = []
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..51b0e41
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,16 @@
+[tool.poetry]
+name = "python-template"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <you@example.com>"]
+
+[tool.poetry.dependencies]
+python = ">=3.8.0,<3.9"
+numpy = "^1.22.2"
+Flask = "^2.2.2"
+
+[tool.poetry.dev-dependencies]
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
\ No newline at end of file
diff --git a/src/database/searchDB.py b/src/database/searchDB.py
new file mode 100644
index 0000000..7413a94
--- /dev/null
+++ b/src/database/searchDB.py
@@ -0,0 +1,66 @@
+class Accessor:
+    _DBFILE = "~/sts-api/.db/search.sqlite3"
+
+    def _refresh(self):
+        if self.buffers.db == None:
+            import sqlite3
+            self.buffers.db = sqlite3.connect(self._DBFILE)
+        self.buffers.dbcursor = self.buffers.db.cursor()
+
+    def list(self, type="*"):
+        self._refresh()
+        self.lastOut = self.buffers.dbcursor.execute(
+            "SELECT {} FROM index".format(type))
+        return self.lastOut
+
+    def search(self, query, limit=None, offset=None):
+        self._refresh()
+        if self.mode == "ro" or self.mode == "rw":
+            import re
+            if query != None:
+                self.search.query = re.compile(f"^*{query}*$", re.IGNORECASE)
+            self.search.limit = limit
+            self.search.offset = offset
+            if limit == None:
+                self.lastOut = re.search(self.search.query, str(self.list()))
+                return self.lastOut
+            else:
+                self.lastOut = re.search(self.search.query, str(self.list()))
+                return self.lastOut[:25]
+
+    def addEntry(self, entryData):
+        if self.mode == "ro":
+            raise ImportError
+        self._refresh()
+        if entryData.url in self.list("url"):
+            raise "EntryExists"
+        self.buffers.dbcursor.execute(f"""
+          INSERT INTO index VALUES
+              ('{entryData.fullURL}', '{entryData.faviconURL}', '{entryData.protocol}', '{entryData.title}', '{entryData.description}')
+        """)
+        self.buffers.db.commit()
+        return True
+    def removeEntry(self, filters):
+      if self.mode == "ro":
+        raise ImportError
+      self._refresh()
+      query="DELETE FROM index WHERE "
+      if filters.full:
+        query+=f"fullURL = {filters.fullURL}"
+      elif filters.hostname:
+        query+=f"fullURL LIKE %{filters.hostname}%"
+
+    query+=";"
+    self.buffers.dbcursor.execute(query)
+    self.buffers.db.commit()
+  return True
+    def __init__(self, mode="ro"):
+        self.mode = mode
+        self.search = {'query': None, 'limit': 20, 'offset': 0}
+        self.lastOut = None
+        self.buffers = {
+            'db': None,
+            'dbcursor': None,
+            'read': None,
+            'write': None
+        }
diff --git a/src/frontend.py b/src/frontend.py
new file mode 100644
index 0000000..d603d73
--- /dev/null
+++ b/src/frontend.py
@@ -0,0 +1,9 @@
+def parsePOST(rqform,src):
+  if rqform.action=="searchQuery":
+    from database import searchDB
+    results=searchDB.Accessor()
+    results.search(query=rqform.query, limit=rqform.limit, offset=rqform.offset)
+    return results.scannerResults
+  elif rqform.action=="genFeed":
+    if rqform.subAction=="operationStatus":
+      
\ No newline at end of file