new stuff
commit
af8e618f01
@ -0,0 +1,8 @@
|
||||
.db
|
||||
./**/__pycache__
|
||||
**.pyc
|
||||
.cache
|
||||
*replit*
|
||||
.config
|
||||
venv
|
||||
.upm
|
||||
@ -0,0 +1,6 @@
|
||||
err:
|
||||
gen:
|
||||
UnknownFeedType: "Unknown Feed"
|
||||
UnknownFeedDescription: "Unknown Feed Description"
|
||||
UnknownFeedName: "Unknown Feed Name"
|
||||
|
||||
@ -0,0 +1,10 @@
|
||||
gen:
|
||||
feed:
|
||||
any:
|
||||
title:
|
||||
search: "Search results for %s"
|
||||
description:
|
||||
search:
|
||||
limited: "The top %d results for the search query '%s'"
|
||||
unlimted: "All search results for the search query '%s'\n\n(WARNING: might be unstable!!!)"
|
||||
|
||||
@ -0,0 +1,13 @@
|
||||
<?xml version='1.0' encoding='UTF-8'?>
|
||||
<data>
|
||||
<result type="{{result.type}}">
|
||||
{% if result.errors %}
|
||||
<errors>
|
||||
{% for item in result.errors %}
|
||||
<error type="{{ result.errors[item].type }}" source="{{ result.errors[item].source }}">{{ result.errors[item].remark }}</error>
|
||||
{% endfor %}
|
||||
</errors>
|
||||
{% endif %}
|
||||
</result>
|
||||
<outbody type="{{ outbody.type }}" compression="{{ outbody.compression if outbody.compression else "none" }}">{{ outbody.remark }}</outbody>
|
||||
</data>
|
||||
@ -0,0 +1,47 @@
|
||||
{# Metadata generator #}
|
||||
<?xml version='1.0' encoding='UTF-8'?>
|
||||
<rss version='2.0'>
|
||||
<channel>
|
||||
{% if result.rss.title %}
|
||||
{% if ask.rss.type == "search" %}
|
||||
<title>{{ strings.gen.feed.any.title.search.format(result.rss.title) }}</title>
|
||||
{% elif ask.rss.type == "status" %}
|
||||
<title>{{ strings.gen.feed.any.title.status }}</title>
|
||||
{% else %}
|
||||
<title>{{ strings.err.gen.UnknownFeedType }}</title>
|
||||
{% endif %}
|
||||
{% else %}
|
||||
<title>{{ strings.err.gen.UnknownFeedName }}</title>
|
||||
{% endif %}
|
||||
{% if result.rss.description %}
|
||||
{% if ask.rss.type == "search" %}
|
||||
{% if ask.query.limit not 0 %}
|
||||
<description>{{ strings.gen.feed.any.description.search.limited.format(ask.query.limit,result.rss.title) }}</description>
|
||||
{% else %}
|
||||
<description>{{ strings.gen.feed.any.description.search.unlimited.format(result.rss.title) }}</description>
|
||||
{% endif %}
|
||||
{% elif ask.rss.type == "status" %}
|
||||
<description>{{ strings.gen.feed.any.description.status }}</description>
|
||||
|
||||
{% else %}
|
||||
<description>{{ strings.err.gen.UnknownFeedDescription }}</description>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% if result.rss.link %}
|
||||
{% if ask.source %}
|
||||
<link>{{ ask.source.urls.goto % result.rss.link }}</link>
|
||||
{% else %}
|
||||
<link>{{ result.rss.link }}</link>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
<builddate>{{ ask.requestDate }}</builddate>
|
||||
<pubdate></pubdate>
|
||||
<language>{{ "%-%".format(ask.lang.lower(),ask.region.lower()) }}</language>
|
||||
<generator>superTinySearch API Frontend v{{ meta.api.version }} using Python Jinja v3</generator>
|
||||
|
||||
{% for item in result.rss.items %}
|
||||
{% include 'xml.blocks.rss2.item.xml.jinja' %}
|
||||
{% endfor %}
|
||||
|
||||
</channel>
|
||||
</rss>
|
||||
@ -0,0 +1,29 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>submitted url!</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>url submitted!</h1>
|
||||
<p>thank you for your submission! <i>[sts]</i> will process your contributions and will add or remove entires based on your input.</p><br><br>
|
||||
<i>to recap, you submitted...</i><br>
|
||||
{% for action in input.actions %}
|
||||
{% if action.what == "scanurl" %}
|
||||
<h2>a URL to scan...</h2>
|
||||
<p>we will scan the page or file at <strong>{{ action.where }}</strong> and add that and any other URLs found from here...</p><br><br>
|
||||
{% elif action.what == "removeurl" %}
|
||||
<h2>a URL to remove from our databases...</h2>
|
||||
{% if action.why == "personal" %}
|
||||
<p>after we confirm that <strong>{{ action.where }}</strong> contains very personal information, we will remove this and blacklist our scanners to prevent this from being scanned ever again...</p>
|
||||
{% elif action.why == "malicious" %}
|
||||
<p>after we confirm that <strong>{{ action.where }}</strong> contains malicious content, we will remove this and blacklist our scanners to prevent this from being scanned ever again...</p>
|
||||
<p>(if this url is exceptionally dangerous, please report this url to it's hosting provider, or to the proper authorities.)</p>
|
||||
{% elif action.why == "false" %}
|
||||
<p>after we confirm that <strong>{{ action.where }}</strong> contains false content, we will remove this and blacklist our scanners to prevent this from being scanned ever again...</p>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
<hr>
|
||||
<i>thaks for contributing to a <a title="free and open source software">foss project!</a></i>
|
||||
</body>
|
||||
</html>
|
||||
@ -0,0 +1,14 @@
|
||||
<item>
|
||||
<title>{{ item.title }}</title>
|
||||
<link>{{ item.link }}</link>
|
||||
<description>{{ item.description|e }}</description>
|
||||
{% if item.author != "None" %}
|
||||
<author>{{ item.author }}</author>
|
||||
{% endif %}
|
||||
{% if item.tags %}
|
||||
<category>{{ item.tags|join('/') }}</category>
|
||||
{% endif %}
|
||||
{% if item.indexDate != "None" %}
|
||||
<pubDate>{{ item.indexDate }}</pubDate>
|
||||
{% endif %}
|
||||
</item>
|
||||
@ -0,0 +1,74 @@
|
||||
exit()
|
||||
|
||||
__version__="0.0"
|
||||
|
||||
from flask import Flask as wserv
|
||||
|
||||
app = wserv('superTinySearch API Runtime')
|
||||
|
||||
from flask import render_template
|
||||
|
||||
import sys
|
||||
debugmode=False
|
||||
if "debug" in sys.argv:
|
||||
debugmode=True
|
||||
del sys
|
||||
|
||||
def makeServerHeader():
|
||||
import sys
|
||||
return f"superTinySearch API/{__version__} (Unix) (Python v{sys.version})"
|
||||
# @app.errorhandler(404)
|
||||
# def not_found(error):
|
||||
# from flask import request as rq
|
||||
# if rq.args.get('returnFormat','') == "xml"
|
||||
|
||||
@app.route("/")
|
||||
def noHumans():
|
||||
from flask import Response
|
||||
resp=Response(
|
||||
response="running",
|
||||
status=200,
|
||||
mimetype="text/plain"
|
||||
)
|
||||
resp.headers["Server"] = makeServerHeader
|
||||
resp.headers["Accept"] = "text/plain"
|
||||
return resp
|
||||
|
||||
|
||||
@app.route("/about")
|
||||
def aboutThis():
|
||||
from flask import Response
|
||||
resp=Response(
|
||||
response=f"superTinySearch API v{__version__}",
|
||||
status=200,
|
||||
mimetype="text/plain"
|
||||
)
|
||||
resp.headers["Server"] = makeServerHeader
|
||||
resp.headers["Accept"] = "text/plain"
|
||||
return resp
|
||||
|
||||
@app.route("/apifront?returnFormat=<rf>&srcHostname=<src>", methods=['GET','POST'])
|
||||
def apiRun(rf = 'xml', src = None):
|
||||
from flask import request as rq
|
||||
if rq.method == 'GET':
|
||||
return 'GET REQUESTS NOT ALLOWED'
|
||||
else:
|
||||
from src import frontend
|
||||
out=frontend.parsePOST(rq.form.to_dict(flat=False),rf,src)
|
||||
del frontend
|
||||
from flask import Response
|
||||
resp = Response(response=out.content, status=500, mime="text/plain")
|
||||
if rf == "xml":
|
||||
resp.headers["Content-Type"] = "text/xml; charset=utf-8"
|
||||
elif rf == "json":
|
||||
resp.headers["Content-Type"] = "application/json; charset=utf-8"
|
||||
else:
|
||||
resp.headers["Content-Type"] = "text/plain; charset=utf-8"
|
||||
resp.headers["Server"] = makeServerHeader()
|
||||
resp.status=200
|
||||
return resp
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(host="0.0.0.0",port=443)
|
||||
else:
|
||||
raise "INVALID USAGE OF API APP"
|
||||
@ -0,0 +1,134 @@
|
||||
[[package]]
|
||||
name = "click"
|
||||
version = "8.1.3"
|
||||
description = "Composable command line interface toolkit"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
colorama = {version = "*", markers = "platform_system == \"Windows\""}
|
||||
|
||||
[[package]]
|
||||
name = "colorama"
|
||||
version = "0.4.6"
|
||||
description = "Cross-platform colored terminal text."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
|
||||
|
||||
[[package]]
|
||||
name = "flask"
|
||||
version = "2.2.2"
|
||||
description = "A simple framework for building complex web applications."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
click = ">=8.0"
|
||||
importlib-metadata = {version = ">=3.6.0", markers = "python_version < \"3.10\""}
|
||||
itsdangerous = ">=2.0"
|
||||
Jinja2 = ">=3.0"
|
||||
Werkzeug = ">=2.2.2"
|
||||
|
||||
[package.extras]
|
||||
async = ["asgiref (>=3.2)"]
|
||||
dotenv = ["python-dotenv"]
|
||||
|
||||
[[package]]
|
||||
name = "importlib-metadata"
|
||||
version = "6.0.0"
|
||||
description = "Read metadata from Python packages"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
zipp = ">=0.5"
|
||||
|
||||
[package.extras]
|
||||
docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"]
|
||||
perf = ["ipython"]
|
||||
testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8", "importlib-resources (>=1.3)"]
|
||||
|
||||
[[package]]
|
||||
name = "itsdangerous"
|
||||
version = "2.1.2"
|
||||
description = "Safely pass data to untrusted environments and back."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[[package]]
|
||||
name = "jinja2"
|
||||
version = "3.1.2"
|
||||
description = "A very fast and expressive template engine."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
MarkupSafe = ">=2.0"
|
||||
|
||||
[package.extras]
|
||||
i18n = ["Babel (>=2.7)"]
|
||||
|
||||
[[package]]
|
||||
name = "markupsafe"
|
||||
version = "2.1.2"
|
||||
description = "Safely add untrusted strings to HTML/XML markup."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[[package]]
|
||||
name = "numpy"
|
||||
version = "1.22.2"
|
||||
description = "NumPy is the fundamental package for array computing with Python."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
|
||||
[[package]]
|
||||
name = "werkzeug"
|
||||
version = "2.2.2"
|
||||
description = "The comprehensive WSGI web application library."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.dependencies]
|
||||
MarkupSafe = ">=2.1.1"
|
||||
|
||||
[package.extras]
|
||||
watchdog = ["watchdog"]
|
||||
|
||||
[[package]]
|
||||
name = "zipp"
|
||||
version = "3.12.0"
|
||||
description = "Backport of pathlib-compatible object wrapper for zip files"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.7"
|
||||
|
||||
[package.extras]
|
||||
docs = ["sphinx (>=3.5)", "jaraco.packaging (>=9)", "rst.linker (>=1.9)", "furo", "sphinx-lint", "jaraco.tidelift (>=1.4)"]
|
||||
testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "flake8 (<5)", "pytest-cov", "pytest-enabler (>=1.3)", "jaraco.itertools", "func-timeout", "jaraco.functools", "more-itertools", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "pytest-flake8"]
|
||||
|
||||
[metadata]
|
||||
lock-version = "1.1"
|
||||
python-versions = ">=3.8.0,<3.9"
|
||||
content-hash = "400928830379454435838ae9f145feaafae43a4a0c4f65edbe32de50c7a3aa9c"
|
||||
|
||||
[metadata.files]
|
||||
click = []
|
||||
colorama = []
|
||||
flask = []
|
||||
importlib-metadata = []
|
||||
itsdangerous = []
|
||||
jinja2 = []
|
||||
markupsafe = []
|
||||
numpy = []
|
||||
werkzeug = []
|
||||
zipp = []
|
||||
@ -0,0 +1,16 @@
|
||||
[tool.poetry]
|
||||
name = "python-template"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = ["Your Name <you@example.com>"]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = ">=3.8.0,<3.9"
|
||||
numpy = "^1.22.2"
|
||||
Flask = "^2.2.2"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core>=1.0.0"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
@ -0,0 +1,53 @@
|
||||
class Accessor:
|
||||
_DBFILE = "~/sts-api/.db/search.sqlite3"
|
||||
|
||||
def _refresh(self):
|
||||
if self.buffers.db == None:
|
||||
import sqlite3
|
||||
self.buffers.db = sqlite3.connect(self._DBFILE)
|
||||
self.buffers.dbcursor = self.buffers.db.cursor()
|
||||
|
||||
def list(self, type="*"):
|
||||
self._refresh()
|
||||
self.lastOut = self.buffers.dbcursor.execute(
|
||||
"SELECT {} FROM index".format(type))
|
||||
return self.lastOut
|
||||
|
||||
def search(self, query, limit=None, offset=None):
|
||||
self._refresh()
|
||||
if self.mode == "ro" or self.mode == "rw":
|
||||
import re
|
||||
if query != None:
|
||||
self.search.query = re.compile(f"^*{query}*$", re.IGNORECASE)
|
||||
self.search.limit = limit
|
||||
self.search.offset = offset
|
||||
if limit == None:
|
||||
self.lastOut = re.search(self.search.query, str(self.list()))
|
||||
return self.lastOut
|
||||
else:
|
||||
self.lastOut = re.search(self.search.query, str(self.list()))
|
||||
return self.lastOut[:25]
|
||||
|
||||
def addEntry(self, entryData):
|
||||
if self.mode == "ro":
|
||||
raise ImportError
|
||||
self._refresh()
|
||||
if entryData.url in self.list("url"):
|
||||
raise "EntryExists"
|
||||
self.buffers.dbcursor.execute(f"""
|
||||
INSERT INTO index VALUES
|
||||
('{entryData.fullURL}', '{entryData.faviconURL}', '{entryData.protocol}', '{entryData.title}', '{entryData.description}')
|
||||
""")
|
||||
self.buffers.db.commit()
|
||||
return True
|
||||
|
||||
def __init__(self, mode="ro"):
|
||||
self.mode = mode
|
||||
self.search = {'query': None, 'limit': 20, 'offset': 0}
|
||||
self.lastOut = None
|
||||
self.buffers = {
|
||||
'db': None,
|
||||
'dbcursor': None,
|
||||
'read': None,
|
||||
'write': None
|
||||
}
|
||||
@ -0,0 +1,9 @@
|
||||
def parsePOST(rqform,src):
|
||||
if rqform.action=="searchQuery":
|
||||
from database import searchDB
|
||||
results=searchDB.Accessor()
|
||||
results.search(query=rqform.query, limit=rqform.limit, offset=rqform.offset)
|
||||
return results.scannerResults
|
||||
elif rqform.action=="genFeed":
|
||||
if rqform.subAction=="operationStatus":
|
||||
|
||||
Loading…
Reference in New Issue