@@ -0,0 +1,333 @@ | |||
# Created by https://www.toptal.com/developers/gitignore/api/pycharm+all,intellij+all,python,virtualenv | |||
# Edit at https://www.toptal.com/developers/gitignore?templates=pycharm+all,intellij+all,python,virtualenv | |||
### Intellij+all ### | |||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider | |||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 | |||
# User-specific stuff | |||
.idea/**/workspace.xml | |||
.idea/**/tasks.xml | |||
.idea/**/usage.statistics.xml | |||
.idea/**/dictionaries | |||
.idea/**/shelf | |||
# AWS User-specific | |||
.idea/**/aws.xml | |||
# Generated files | |||
.idea/**/contentModel.xml | |||
# Sensitive or high-churn files | |||
.idea/**/dataSources/ | |||
.idea/**/dataSources.ids | |||
.idea/**/dataSources.local.xml | |||
.idea/**/sqlDataSources.xml | |||
.idea/**/dynamic.xml | |||
.idea/**/uiDesigner.xml | |||
.idea/**/dbnavigator.xml | |||
# Gradle | |||
.idea/**/gradle.xml | |||
.idea/**/libraries | |||
# Gradle and Maven with auto-import | |||
# When using Gradle or Maven with auto-import, you should exclude module files, | |||
# since they will be recreated, and may cause churn. Uncomment if using | |||
# auto-import. | |||
# .idea/artifacts | |||
# .idea/compiler.xml | |||
# .idea/jarRepositories.xml | |||
# .idea/modules.xml | |||
# .idea/*.iml | |||
# .idea/modules | |||
# *.iml | |||
# *.ipr | |||
# CMake | |||
cmake-build-*/ | |||
# Mongo Explorer plugin | |||
.idea/**/mongoSettings.xml | |||
# File-based project format | |||
*.iws | |||
# IntelliJ | |||
out/ | |||
# mpeltonen/sbt-idea plugin | |||
.idea_modules/ | |||
# JIRA plugin | |||
atlassian-ide-plugin.xml | |||
# Cursive Clojure plugin | |||
.idea/replstate.xml | |||
# SonarLint plugin | |||
.idea/sonarlint/ | |||
# Crashlytics plugin (for Android Studio and IntelliJ) | |||
com_crashlytics_export_strings.xml | |||
crashlytics.properties | |||
crashlytics-build.properties | |||
fabric.properties | |||
# Editor-based Rest Client | |||
.idea/httpRequests | |||
# Android studio 3.1+ serialized cache file | |||
.idea/caches/build_file_checksums.ser | |||
### Intellij+all Patch ### | |||
# Ignore everything but code style settings and run configurations | |||
# that are supposed to be shared within teams. | |||
.idea/* | |||
!.idea/codeStyles | |||
!.idea/runConfigurations | |||
### PyCharm+all ### | |||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider | |||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 | |||
# User-specific stuff | |||
# AWS User-specific | |||
# Generated files | |||
# Sensitive or high-churn files | |||
# Gradle | |||
# Gradle and Maven with auto-import | |||
# When using Gradle or Maven with auto-import, you should exclude module files, | |||
# since they will be recreated, and may cause churn. Uncomment if using | |||
# auto-import. | |||
# .idea/artifacts | |||
# .idea/compiler.xml | |||
# .idea/jarRepositories.xml | |||
# .idea/modules.xml | |||
# .idea/*.iml | |||
# .idea/modules | |||
# *.iml | |||
# *.ipr | |||
# CMake | |||
# Mongo Explorer plugin | |||
# File-based project format | |||
# IntelliJ | |||
# mpeltonen/sbt-idea plugin | |||
# JIRA plugin | |||
# Cursive Clojure plugin | |||
# SonarLint plugin | |||
# Crashlytics plugin (for Android Studio and IntelliJ) | |||
# Editor-based Rest Client | |||
# Android studio 3.1+ serialized cache file | |||
### PyCharm+all Patch ### | |||
# Ignore everything but code style settings and run configurations | |||
# that are supposed to be shared within teams. | |||
### Python ### | |||
# Byte-compiled / optimized / DLL files | |||
__pycache__/ | |||
*.py[cod] | |||
*$py.class | |||
# C extensions | |||
*.so | |||
# Distribution / packaging | |||
.Python | |||
build/ | |||
develop-eggs/ | |||
dist/ | |||
downloads/ | |||
eggs/ | |||
.eggs/ | |||
lib/ | |||
lib64/ | |||
parts/ | |||
sdist/ | |||
var/ | |||
wheels/ | |||
share/python-wheels/ | |||
*.egg-info/ | |||
.installed.cfg | |||
*.egg | |||
MANIFEST | |||
# PyInstaller | |||
# Usually these files are written by a python script from a template | |||
# before PyInstaller builds the exe, so as to inject date/other infos into it. | |||
*.manifest | |||
*.spec | |||
# Installer logs | |||
pip-log.txt | |||
pip-delete-this-directory.txt | |||
# Unit test / coverage reports | |||
htmlcov/ | |||
.tox/ | |||
.nox/ | |||
.coverage | |||
.coverage.* | |||
.cache | |||
nosetests.xml | |||
coverage.xml | |||
*.cover | |||
*.py,cover | |||
.hypothesis/ | |||
.pytest_cache/ | |||
cover/ | |||
# Translations | |||
*.mo | |||
*.pot | |||
# Django stuff: | |||
*.log | |||
local_settings.py | |||
db.sqlite3 | |||
db.sqlite3-journal | |||
# Flask stuff: | |||
instance/ | |||
.webassets-cache | |||
# Scrapy stuff: | |||
.scrapy | |||
# Sphinx documentation | |||
docs/_build/ | |||
# PyBuilder | |||
.pybuilder/ | |||
target/ | |||
# Jupyter Notebook | |||
.ipynb_checkpoints | |||
# IPython | |||
profile_default/ | |||
ipython_config.py | |||
# pyenv | |||
# For a library or package, you might want to ignore these files since the code is | |||
# intended to run in multiple environments; otherwise, check them in: | |||
# .python-version | |||
# pipenv | |||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. | |||
# However, in case of collaboration, if having platform-specific dependencies or dependencies | |||
# having no cross-platform support, pipenv may install dependencies that don't work, or not | |||
# install all needed dependencies. | |||
#Pipfile.lock | |||
# poetry | |||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. | |||
# This is especially recommended for binary packages to ensure reproducibility, and is more | |||
# commonly ignored for libraries. | |||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control | |||
#poetry.lock | |||
# pdm | |||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. | |||
#pdm.lock | |||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it | |||
# in version control. | |||
# https://pdm.fming.dev/#use-with-ide | |||
.pdm.toml | |||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm | |||
__pypackages__/ | |||
# Celery stuff | |||
celerybeat-schedule | |||
celerybeat.pid | |||
# SageMath parsed files | |||
*.sage.py | |||
# Environments | |||
.env | |||
.venv | |||
env/ | |||
venv/ | |||
ENV/ | |||
env.bak/ | |||
venv.bak/ | |||
# Spyder project settings | |||
.spyderproject | |||
.spyproject | |||
# Rope project settings | |||
.ropeproject | |||
# mkdocs documentation | |||
/site | |||
# mypy | |||
.mypy_cache/ | |||
.dmypy.json | |||
dmypy.json | |||
# Pyre type checker | |||
.pyre/ | |||
# pytype static type analyzer | |||
.pytype/ | |||
# Cython debug symbols | |||
cython_debug/ | |||
# PyCharm | |||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can | |||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore | |||
# and can be added to the global gitignore or merged into this file. For a more nuclear | |||
# option (not recommended) you can uncomment the following to ignore the entire idea folder. | |||
#.idea/ | |||
### Python Patch ### | |||
# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration | |||
poetry.toml | |||
# ruff | |||
.ruff_cache/ | |||
# LSP config files | |||
pyrightconfig.json | |||
### VirtualEnv ### | |||
# Virtualenv | |||
# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ | |||
[Bb]in | |||
[Ii]nclude | |||
[Ll]ib | |||
[Ll]ib64 | |||
[Ll]ocal | |||
[Ss]cripts | |||
pyvenv.cfg | |||
pip-selfcheck.json | |||
# End of https://www.toptal.com/developers/gitignore/api/pycharm+all,intellij+all,python,virtualenv | |||
/venv | |||
/config.py |
@@ -0,0 +1,18 @@ | |||
--- | |||
kind: pipeline | |||
name: default | |||
steps: | |||
- name: docker | |||
image: plugins/docker | |||
settings: | |||
registry: atdr-writer.meo.ws | |||
username: | |||
from_secret: atdr_user | |||
password: | |||
from_secret: atdr_pass | |||
repo: atdr-writer.meo.ws/archiveteam/offload-dispatcher | |||
dockerfile: Dockerfile | |||
purge: true | |||
auto_tag: false | |||
tags: | |||
- latest |
@@ -0,0 +1,333 @@ | |||
# Created by https://www.toptal.com/developers/gitignore/api/pycharm+all,intellij+all,python,virtualenv | |||
# Edit at https://www.toptal.com/developers/gitignore?templates=pycharm+all,intellij+all,python,virtualenv | |||
### Intellij+all ### | |||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider | |||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 | |||
# User-specific stuff | |||
.idea/**/workspace.xml | |||
.idea/**/tasks.xml | |||
.idea/**/usage.statistics.xml | |||
.idea/**/dictionaries | |||
.idea/**/shelf | |||
# AWS User-specific | |||
.idea/**/aws.xml | |||
# Generated files | |||
.idea/**/contentModel.xml | |||
# Sensitive or high-churn files | |||
.idea/**/dataSources/ | |||
.idea/**/dataSources.ids | |||
.idea/**/dataSources.local.xml | |||
.idea/**/sqlDataSources.xml | |||
.idea/**/dynamic.xml | |||
.idea/**/uiDesigner.xml | |||
.idea/**/dbnavigator.xml | |||
# Gradle | |||
.idea/**/gradle.xml | |||
.idea/**/libraries | |||
# Gradle and Maven with auto-import | |||
# When using Gradle or Maven with auto-import, you should exclude module files, | |||
# since they will be recreated, and may cause churn. Uncomment if using | |||
# auto-import. | |||
# .idea/artifacts | |||
# .idea/compiler.xml | |||
# .idea/jarRepositories.xml | |||
# .idea/modules.xml | |||
# .idea/*.iml | |||
# .idea/modules | |||
# *.iml | |||
# *.ipr | |||
# CMake | |||
cmake-build-*/ | |||
# Mongo Explorer plugin | |||
.idea/**/mongoSettings.xml | |||
# File-based project format | |||
*.iws | |||
# IntelliJ | |||
out/ | |||
# mpeltonen/sbt-idea plugin | |||
.idea_modules/ | |||
# JIRA plugin | |||
atlassian-ide-plugin.xml | |||
# Cursive Clojure plugin | |||
.idea/replstate.xml | |||
# SonarLint plugin | |||
.idea/sonarlint/ | |||
# Crashlytics plugin (for Android Studio and IntelliJ) | |||
com_crashlytics_export_strings.xml | |||
crashlytics.properties | |||
crashlytics-build.properties | |||
fabric.properties | |||
# Editor-based Rest Client | |||
.idea/httpRequests | |||
# Android studio 3.1+ serialized cache file | |||
.idea/caches/build_file_checksums.ser | |||
### Intellij+all Patch ### | |||
# Ignore everything but code style settings and run configurations | |||
# that are supposed to be shared within teams. | |||
.idea/* | |||
!.idea/codeStyles | |||
!.idea/runConfigurations | |||
### PyCharm+all ### | |||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider | |||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 | |||
# User-specific stuff | |||
# AWS User-specific | |||
# Generated files | |||
# Sensitive or high-churn files | |||
# Gradle | |||
# Gradle and Maven with auto-import | |||
# When using Gradle or Maven with auto-import, you should exclude module files, | |||
# since they will be recreated, and may cause churn. Uncomment if using | |||
# auto-import. | |||
# .idea/artifacts | |||
# .idea/compiler.xml | |||
# .idea/jarRepositories.xml | |||
# .idea/modules.xml | |||
# .idea/*.iml | |||
# .idea/modules | |||
# *.iml | |||
# *.ipr | |||
# CMake | |||
# Mongo Explorer plugin | |||
# File-based project format | |||
# IntelliJ | |||
# mpeltonen/sbt-idea plugin | |||
# JIRA plugin | |||
# Cursive Clojure plugin | |||
# SonarLint plugin | |||
# Crashlytics plugin (for Android Studio and IntelliJ) | |||
# Editor-based Rest Client | |||
# Android studio 3.1+ serialized cache file | |||
### PyCharm+all Patch ### | |||
# Ignore everything but code style settings and run configurations | |||
# that are supposed to be shared within teams. | |||
### Python ### | |||
# Byte-compiled / optimized / DLL files | |||
__pycache__/ | |||
*.py[cod] | |||
*$py.class | |||
# C extensions | |||
*.so | |||
# Distribution / packaging | |||
.Python | |||
build/ | |||
develop-eggs/ | |||
dist/ | |||
downloads/ | |||
eggs/ | |||
.eggs/ | |||
lib/ | |||
lib64/ | |||
parts/ | |||
sdist/ | |||
var/ | |||
wheels/ | |||
share/python-wheels/ | |||
*.egg-info/ | |||
.installed.cfg | |||
*.egg | |||
MANIFEST | |||
# PyInstaller | |||
# Usually these files are written by a python script from a template | |||
# before PyInstaller builds the exe, so as to inject date/other infos into it. | |||
*.manifest | |||
*.spec | |||
# Installer logs | |||
pip-log.txt | |||
pip-delete-this-directory.txt | |||
# Unit test / coverage reports | |||
htmlcov/ | |||
.tox/ | |||
.nox/ | |||
.coverage | |||
.coverage.* | |||
.cache | |||
nosetests.xml | |||
coverage.xml | |||
*.cover | |||
*.py,cover | |||
.hypothesis/ | |||
.pytest_cache/ | |||
cover/ | |||
# Translations | |||
*.mo | |||
*.pot | |||
# Django stuff: | |||
*.log | |||
local_settings.py | |||
db.sqlite3 | |||
db.sqlite3-journal | |||
# Flask stuff: | |||
instance/ | |||
.webassets-cache | |||
# Scrapy stuff: | |||
.scrapy | |||
# Sphinx documentation | |||
docs/_build/ | |||
# PyBuilder | |||
.pybuilder/ | |||
target/ | |||
# Jupyter Notebook | |||
.ipynb_checkpoints | |||
# IPython | |||
profile_default/ | |||
ipython_config.py | |||
# pyenv | |||
# For a library or package, you might want to ignore these files since the code is | |||
# intended to run in multiple environments; otherwise, check them in: | |||
# .python-version | |||
# pipenv | |||
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. | |||
# However, in case of collaboration, if having platform-specific dependencies or dependencies | |||
# having no cross-platform support, pipenv may install dependencies that don't work, or not | |||
# install all needed dependencies. | |||
#Pipfile.lock | |||
# poetry | |||
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. | |||
# This is especially recommended for binary packages to ensure reproducibility, and is more | |||
# commonly ignored for libraries. | |||
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control | |||
#poetry.lock | |||
# pdm | |||
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. | |||
#pdm.lock | |||
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it | |||
# in version control. | |||
# https://pdm.fming.dev/#use-with-ide | |||
.pdm.toml | |||
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm | |||
__pypackages__/ | |||
# Celery stuff | |||
celerybeat-schedule | |||
celerybeat.pid | |||
# SageMath parsed files | |||
*.sage.py | |||
# Environments | |||
.env | |||
.venv | |||
env/ | |||
venv/ | |||
ENV/ | |||
env.bak/ | |||
venv.bak/ | |||
# Spyder project settings | |||
.spyderproject | |||
.spyproject | |||
# Rope project settings | |||
.ropeproject | |||
# mkdocs documentation | |||
/site | |||
# mypy | |||
.mypy_cache/ | |||
.dmypy.json | |||
dmypy.json | |||
# Pyre type checker | |||
.pyre/ | |||
# pytype static type analyzer | |||
.pytype/ | |||
# Cython debug symbols | |||
cython_debug/ | |||
# PyCharm | |||
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can | |||
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore | |||
# and can be added to the global gitignore or merged into this file. For a more nuclear | |||
# option (not recommended) you can uncomment the following to ignore the entire idea folder. | |||
#.idea/ | |||
### Python Patch ### | |||
# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration | |||
poetry.toml | |||
# ruff | |||
.ruff_cache/ | |||
# LSP config files | |||
pyrightconfig.json | |||
### VirtualEnv ### | |||
# Virtualenv | |||
# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ | |||
[Bb]in | |||
[Ii]nclude | |||
[Ll]ib | |||
[Ll]ib64 | |||
[Ll]ocal | |||
[Ss]cripts | |||
pyvenv.cfg | |||
pip-selfcheck.json | |||
# End of https://www.toptal.com/developers/gitignore/api/pycharm+all,intellij+all,python,virtualenv | |||
/venv | |||
/config.py |
@@ -0,0 +1,14 @@ | |||
FROM python:3.11-bookworm | |||
RUN apt update && apt install -y tini ca-certificates && \ | |||
apt-get clean && \ | |||
rm -rf /var/lib/apt/lists/* | |||
WORKDIR /dispatcher | |||
COPY requirements.txt . | |||
RUN pip install -r requirements.txt | |||
COPY main.py . | |||
ENV DISPATCHER_SETTINGS=/dispatcher/config.py | |||
ENTRYPOINT [ "/usr/bin/tini-static", "--", "gunicorn", "-w", "4", "--access-logfile=-", "main:app" ] |
@@ -0,0 +1,12 @@ | |||
version: "3.7" | |||
services: | |||
dispatcher: | |||
build: | |||
context: . | |||
volumes: | |||
- './config.py:/dispatcher/config.py:ro' | |||
ports: | |||
- '127.0.0.1:8000:8000' | |||
command: | |||
- '-b' | |||
- '0.0.0.0' |
@@ -0,0 +1,22 @@ | |||
PROMETHEUS = { | |||
"url": "http://127.0.0.1:9090", | |||
"disable_ssl": True, | |||
} | |||
# URL FORMATS: | |||
# | |||
# Backend # TLS # Bucket/item # URL Format | |||
########################################### | |||
# Minio S3 # N # Y # minio+http://user:password@host[:port] | |||
# Minio S3 # Y # Y # minio+https://user:password@host[:port] | |||
TARGETS = [ | |||
{ | |||
"url": "minio+https://user:password@minio", | |||
"free_space": { | |||
"query": "minio_cluster_capacity_usable_free_bytes{job=\"somejob\"}", | |||
"minimum": (1024 * 1024 * 1024 * 500), # 500 gigabytes | |||
} | |||
}, | |||
] |
@@ -0,0 +1,38 @@ | |||
import copy | |||
from flask import Flask, request | |||
from prometheus_api_client import PrometheusConnect | |||
import random | |||
app = Flask(__name__) | |||
app.config.from_envvar('DISPATCHER_SETTINGS') | |||
def get_prom_client(): | |||
return PrometheusConnect(**app.config["PROMETHEUS"]) | |||
@app.route("/") | |||
def hello_world(): | |||
return "Target offload dispatcher" | |||
@app.route("/offload_target") | |||
def offload_target(): | |||
expected_size = int(request.args.get("SIZE_HINT", 20*1024*1024*1024)) | |||
targets = copy.deepcopy(app.config["TARGETS"]) | |||
random.shuffle(targets) | |||
prom = get_prom_client() | |||
for target in targets: | |||
app.logger.info(f"Considering target {target['url']}...") | |||
result = prom.custom_query(query=target["free_space"]["query"]) | |||
if len(result) != 1: | |||
continue | |||
result = result[0] | |||
target_free_space = int(result['value'][1]) | |||
expected_free_space = target_free_space - expected_size | |||
app.logger.info(f"Available space: {target_free_space} Expected size: {expected_size}") | |||
if expected_free_space < target["free_space"].get("minimum", 1024 * 1024 * 1024 * 500): | |||
continue | |||
return {"url": target["url"]} | |||
return "Unable to allocate target", 500 |
@@ -0,0 +1,3 @@ | |||
flask~=2.3.3 | |||
gunicorn | |||
prometheus-api-client |