elburg/sts-api
Template
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

new stuff

Browse source
This commit is contained in:
elburg 2023-02-02 02:47:12 +00:00
commit af8e618f01
12 changed files with 413 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.gitignore vendored Normal file
View file

@ -0,0 +1,8 @@
.db
./**/__pycache__
**.pyc
.cache
*replit*
.config
venv
.upm

6
data/locale/en_US.err.yaml Normal file
View file

@ -0,0 +1,6 @@
err:
gen:
UnknownFeedType: "Unknown Feed"
UnknownFeedDescription: "Unknown Feed Description"
UnknownFeedName: "Unknown Feed Name"

10
data/locale/en_US.gen.feed.yaml Normal file
View file

@ -0,0 +1,10 @@
gen:
feed:
any:
title:
search: "Search results for %s"
description:
search:
limited: "The top %d results for the search query '%s'"
unlimted: "All search results for the search query '%s'\n\n(WARNING: might be unstable!!!)"

13
data/templates/errorpage.404.basic.xml.jinja Normal file
View file

@ -0,0 +1,13 @@
<?xml version='1.0' encoding='UTF-8'?>
<data>
<result type="{{result.type}}">
{% if result.errors %}
<errors>
{% for item in result.errors %}
<error type="{{ result.errors[item].type }}" source="{{ result.errors[item].source }}">{{ result.errors[item].remark }}</error>
{% endfor %}
</errors>
{% endif %}
</result>
<outbody type="{{ outbody.type }}" compression="{{ outbody.compression if outbody.compression else "none" }}">{{ outbody.remark }}</outbody>
</data>

47
data/templates/feed.rss2.xml.jinja Normal file
View file

@ -0,0 +1,47 @@
{# Metadata generator #}
<?xml version='1.0' encoding='UTF-8'?>
<rss version='2.0'>
<channel>
{% if result.rss.title %}
{% if ask.rss.type == "search" %}
<title>{{ strings.gen.feed.any.title.search.format(result.rss.title) }}</title>
{% elif ask.rss.type == "status" %}
<title>{{ strings.gen.feed.any.title.status }}</title>
{% else %}
<title>{{ strings.err.gen.UnknownFeedType }}</title>
{% endif %}
{% else %}
<title>{{ strings.err.gen.UnknownFeedName }}</title>
{% endif %}
{% if result.rss.description %}
{% if ask.rss.type == "search" %}
{% if ask.query.limit not 0 %}
<description>{{ strings.gen.feed.any.description.search.limited.format(ask.query.limit,result.rss.title) }}</description>
{% else %}
<description>{{ strings.gen.feed.any.description.search.unlimited.format(result.rss.title) }}</description>
{% endif %}
{% elif ask.rss.type == "status" %}
<description>{{ strings.gen.feed.any.description.status }}</description>
{% else %}
<description>{{ strings.err.gen.UnknownFeedDescription }}</description>
{% endif %}
{% endif %}
{% if result.rss.link %}
{% if ask.source %}
<link>{{ ask.source.urls.goto % result.rss.link }}</link>
{% else %}
<link>{{ result.rss.link }}</link>
{% endif %}
{% endif %}
<builddate>{{ ask.requestDate }}</builddate>
<pubdate></pubdate>
<language>{{ "%-%".format(ask.lang.lower(),ask.region.lower()) }}</language>
<generator>superTinySearch API Frontend v{{ meta.api.version }} using Python Jinja v3</generator>
{% for item in result.rss.items %}
{% include 'xml.blocks.rss2.item.xml.jinja' %}
{% endfor %}
</channel>
</rss>

29
data/templates/hidden.commServ.submitURL.ok.html.jinja Normal file
View file

@ -0,0 +1,29 @@
<!DOCTYPE html>
<html>
<head>
<title>submitted url!</title>
</head>
<body>
<h1>url submitted!</h1>
<p>thank you for your submission! <i>[sts]</i> will process your contributions and will add or remove entires based on your input.</p><br><br>
<i>to recap, you submitted...</i><br>
{% for action in input.actions %}
{% if action.what == "scanurl" %}
<h2>a URL to scan...</h2>
<p>we will scan the page or file at <strong>{{ action.where }}</strong> and add that and any other URLs found from here...</p><br><br>
{% elif action.what == "removeurl" %}
<h2>a URL to remove from our databases...</h2>
{% if action.why == "personal" %}
<p>after we confirm that <strong>{{ action.where }}</strong> contains very personal information, we will remove this and blacklist our scanners to prevent this from being scanned ever again...</p>
{% elif action.why == "malicious" %}
<p>after we confirm that <strong>{{ action.where }}</strong> contains malicious content, we will remove this and blacklist our scanners to prevent this from being scanned ever again...</p>
<p>(if this url is exceptionally dangerous, please report this url to it's hosting provider, or to the proper authorities.)</p>
{% elif action.why == "false" %}
<p>after we confirm that <strong>{{ action.where }}</strong> contains false content, we will remove this and blacklist our scanners to prevent this from being scanned ever again...</p>
{% endif %}
{% endif %}
{% endfor %}
<hr>
<i>thaks for contributing to a <a title="free and open source software">foss project!</a></i>
</body>
</html>

14
data/templates/xml.blocks.rss2.item.xml.jinja Normal file
View file

@ -0,0 +1,14 @@
<item>
<title>{{ item.title }}</title>
<link>{{ item.link }}</link>
<description>{{ item.description|e }}</description>
{% if item.author != "None" %}
<author>{{ item.author }}</author>
{% endif %}
{% if item.tags %}
<category>{{ item.tags|join('/') }}</category>
{% endif %}
{% if item.indexDate != "None" %}
<pubDate>{{ item.indexDate }}</pubDate>
{% endif %}
</item>

74
main.py Normal file
View file

@ -0,0 +1,74 @@
exit()
__version__="0.0"
from flask import Flask as wserv
app = wserv('superTinySearch API Runtime')
from flask import render_template
import sys
debugmode=False
if "debug" in sys.argv:
debugmode=True
del sys
def makeServerHeader():
import sys
return f"superTinySearch API/{__version__} (Unix) (Python v{sys.version})"
# @app.errorhandler(404)
# def not_found(error):
# from flask import request as rq
# if rq.args.get('returnFormat','') == "xml"
@app.route("/")
def noHumans():
from flask import Response
resp=Response(
response="running",
status=200,
mimetype="text/plain"
)
resp.headers["Server"] = makeServerHeader
resp.headers["Accept"] = "text/plain"
return resp
@app.route("/about")
def aboutThis():
from flask import Response
resp=Response(
response=f"superTinySearch API v{__version__}",
status=200,
mimetype="text/plain"
)
resp.headers["Server"] = makeServerHeader
resp.headers["Accept"] = "text/plain"
return resp
@app.route("/apifront?returnFormat=<rf>&srcHostname=<src>", methods=['GET','POST'])
def apiRun(rf = 'xml', src = None):
from flask import request as rq
if rq.method == 'GET':
return 'GET REQUESTS NOT ALLOWED'
else:
from src import frontend
out=frontend.parsePOST(rq.form.to_dict(flat=False),rf,src)
del frontend
from flask import Response
resp = Response(response=out.content, status=500, mime="text/plain")
if rf == "xml":
resp.headers["Content-Type"] = "text/xml; charset=utf-8"
elif rf == "json":
resp.headers["Content-Type"] = "application/json; charset=utf-8"
else:
resp.headers["Content-Type"] = "text/plain; charset=utf-8"
resp.headers["Server"] = makeServerHeader()
resp.status=200
return resp
if __name__ == '__main__':
app.run(host="0.0.0.0",port=443)
else:
raise "INVALID USAGE OF API APP"

134
poetry.lock generated Normal file
View file

@ -0,0 +1,134 @@
[[package]]
name = "click"
version = "8.1.3"
description = "Composable command line interface toolkit"
category = "main"
optional = false
python-versions = ">=3.7"
[package.dependencies]
colorama = {version = "*", markers = "platform_system == \"Windows\""}
[[package]]
name = "colorama"
version = "0.4.6"
description = "Cross-platform colored terminal text."
category = "main"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
[[package]]
name = "flask"
version = "2.2.2"
description = "A simple framework for building complex web applications."
category = "main"
optional = false
python-versions = ">=3.7"
[package.dependencies]
click = ">=8.0"
importlib-metadata = {version = ">=3.6.0", markers = "python_version < \"3.10\""}
itsdangerous = ">=2.0"
Jinja2 = ">=3.0"
Werkzeug = ">=2.2.2"
[package.extras]
async = ["asgiref (>=3.2)"]
dotenv = ["python-dotenv"]
[[package]]
name = "importlib-metadata"
version = "6.0.0"
description = "Read metadata from Python packages"
category = "main"
optional = false
python-versions = ">=3.7"
[package.dependencies]
zipp = ">=0.5"
[package.extras]
docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"]
perf = ["ipython"]
testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8", "importlib-resources (>=1.3)"]
[[package]]
name = "itsdangerous"
version = "2.1.2"
description = "Safely pass data to untrusted environments and back."
category = "main"
optional = false
python-versions = ">=3.7"
[[package]]
name = "jinja2"
version = "3.1.2"
description = "A very fast and expressive template engine."
category = "main"
optional = false
python-versions = ">=3.7"
[package.dependencies]
MarkupSafe = ">=2.0"
[package.extras]
i18n = ["Babel (>=2.7)"]
[[package]]
name = "markupsafe"
version = "2.1.2"
description = "Safely add untrusted strings to HTML/XML markup."
category = "main"
optional = false
python-versions = ">=3.7"
[[package]]
name = "numpy"
version = "1.22.2"
description = "NumPy is the fundamental package for array computing with Python."
category = "main"
optional = false
python-versions = ">=3.8"
[[package]]
name = "werkzeug"
version = "2.2.2"
description = "The comprehensive WSGI web application library."
category = "main"
optional = false
python-versions = ">=3.7"
[package.dependencies]
MarkupSafe = ">=2.1.1"
[package.extras]
watchdog = ["watchdog"]
[[package]]
name = "zipp"
version = "3.12.0"
description = "Backport of pathlib-compatible object wrapper for zip files"
category = "main"
optional = false
python-versions = ">=3.7"
[package.extras]
docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"]
testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "jaraco.functools", "more-itertools", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"]
[metadata]
lock-version = "1.1"
python-versions = ">=3.8.0,<3.9"
content-hash = "400928830379454435838ae9f145feaafae43a4a0c4f65edbe32de50c7a3aa9c"
[metadata.files]
click = []
colorama = []
flask = []
importlib-metadata = []
itsdangerous = []
jinja2 = []
markupsafe = []
numpy = []
werkzeug = []
zipp = []

16
pyproject.toml Normal file
View file

@ -0,0 +1,16 @@
[tool.poetry]
name = "python-template"
version = "0.1.0"
description = ""
authors = ["Your Name <you@example.com>"]
[tool.poetry.dependencies]
python = ">=3.8.0,<3.9"
numpy = "^1.22.2"
Flask = "^2.2.2"
[tool.poetry.dev-dependencies]
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

53
src/database/searchDB.py Normal file
View file

@ -0,0 +1,53 @@
class Accessor:
_DBFILE = "~/sts-api/.db/search.sqlite3"
def _refresh(self):
if self.buffers.db == None:
import sqlite3
self.buffers.db = sqlite3.connect(self._DBFILE)
self.buffers.dbcursor = self.buffers.db.cursor()
def list(self, type="*"):
self._refresh()
self.lastOut = self.buffers.dbcursor.execute(
"SELECT {} FROM index".format(type))
return self.lastOut
def search(self, query, limit=None, offset=None):
self._refresh()
if self.mode == "ro" or self.mode == "rw":
import re
if query != None:
self.search.query = re.compile(f"^*{query}*$", re.IGNORECASE)
self.search.limit = limit
self.search.offset = offset
if limit == None:
self.lastOut = re.search(self.search.query, str(self.list()))
return self.lastOut
else:
self.lastOut = re.search(self.search.query, str(self.list()))
return self.lastOut[:25]
def addEntry(self, entryData):
if self.mode == "ro":
raise ImportError
self._refresh()
if entryData.url in self.list("url"):
raise "EntryExists"
self.buffers.dbcursor.execute(f"""
INSERT INTO index VALUES
('{entryData.fullURL}', '{entryData.faviconURL}', '{entryData.protocol}', '{entryData.title}', '{entryData.description}')
""")
self.buffers.db.commit()
return True
def __init__(self, mode="ro"):
self.mode = mode
self.search = {'query': None, 'limit': 20, 'offset': 0}
self.lastOut = None
self.buffers = {
'db': None,
'dbcursor': None,
'read': None,
'write': None
}

9
src/frontend.py Normal file
View file

@ -0,0 +1,9 @@
def parsePOST(rqform,src):
if rqform.action=="searchQuery":
from database import searchDB
results=searchDB.Accessor()
results.search(query=rqform.query, limit=rqform.limit, offset=rqform.offset)
return results.scannerResults
elif rqform.action=="genFeed":
if rqform.subAction=="operationStatus":