From eba9b410c6da6ad5ba83197b58144c77cbc300a8 Mon Sep 17 00:00:00 2001 From: clonejo Date: Sun, 14 Jan 2024 18:56:43 +0100 Subject: [PATCH 01/12] autoformat by black+isort --- crud.py | 13 +++++++++---- main.py | 18 ++++++++++++------ models.py | 3 ++- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/crud.py b/crud.py index f5e1f39..6f453f2 100644 --- a/crud.py +++ b/crud.py @@ -1,14 +1,19 @@ -from sqlalchemy.orm import Session -from sqlalchemy import DECIMAL, cast -import models, schemas +from sqlalchemy import DECIMAL, cast +from sqlalchemy.orm import Session + +import models +import schemas + def get_items(db: Session) -> list[models.Item]: return db.query(models.Item).all() def put_item(db: Session, id: str, item: schemas.Item): - updated = bool(db.query(models.Item).filter(models.Item.id == id).update(item.dict())) + updated = bool( + db.query(models.Item).filter(models.Item.id == id).update(item.dict()) + ) if not updated: db.add(models.Item(**item.dict())) db.commit() diff --git a/main.py b/main.py index ecaeea6..dbc0409 100644 --- a/main.py +++ b/main.py @@ -1,9 +1,10 @@ +import re + from fastapi import Depends, FastAPI, Response from fastapi.staticfiles import StaticFiles from sqlalchemy import event -from sqlalchemy.orm import Session from sqlalchemy.engine import Engine -import re +from sqlalchemy.orm import Session import crud from database import SessionLocal, engine @@ -14,6 +15,7 @@ Base.metadata.create_all(bind=engine) app = FastAPI() + def get_db(): db = SessionLocal() try: @@ -21,6 +23,7 @@ def get_db(): finally: db.close() + @event.listens_for(Engine, "connect") def _set_sqlite_pragma(conn, _): cursor = conn.cursor() @@ -31,21 +34,24 @@ def _set_sqlite_pragma(conn, _): @app.get("/api/items", response_model=dict[str, Item]) async def list_items(db: Session = Depends(get_db)): # natural sort by id - natsort = lambda item: [int(t) if t.isdigit() else t.lower() for t in re.split('(\d+)', item.id)] + natsort = lambda item: [ + int(t) if t.isdigit() else t.lower() for t in re.split("(\d+)", item.id) + ] items = crud.get_items(db) items = sorted(items, key=natsort) - return {i.id:i for i in items} + return {i.id: i for i in items} @app.put("/api/items/{id}") async def put_item(id: str, item: Item, db: Session = Depends(get_db)): if crud.put_item(db, id, item): - return Response(b'', status_code=204) - return Response(b'', status_code=201) + return Response(b"", status_code=204) + return Response(b"", status_code=201) @app.delete("/api/items/{id}", status_code=204) async def delete_item(id: str, db: Session = Depends(get_db)): crud.delete_item(db, id) + app.mount("/", StaticFiles(directory="static", html=True), name="static") diff --git a/models.py b/models.py index d7618a6..7aa9b21 100644 --- a/models.py +++ b/models.py @@ -1,7 +1,8 @@ -from sqlalchemy import Column, ForeignKey, String, Text, Boolean +from sqlalchemy import Boolean, Column, ForeignKey, String, Text from database import Base + class Item(Base): __tablename__ = "items" From 99df802a40144f41674c8bfb453c66daee2c5b71 Mon Sep 17 00:00:00 2001 From: clonejo Date: Sun, 14 Jan 2024 18:57:32 +0100 Subject: [PATCH 02/12] make return value of crud.put_item easier to understand --- crud.py | 11 ++++++++++- main.py | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/crud.py b/crud.py index 6f453f2..277c881 100644 --- a/crud.py +++ b/crud.py @@ -1,3 +1,4 @@ +from enum import Enum from sqlalchemy import DECIMAL, cast from sqlalchemy.orm import Session @@ -17,7 +18,15 @@ def put_item(db: Session, id: str, item: schemas.Item): if not updated: db.add(models.Item(**item.dict())) db.commit() - return updated + if updated: + return PutItemResult.UPDATED + else: + return PutItemResult.ADDED + + +class PutItemResult(Enum): + ADDED = 1 + UPDATED = 2 def delete_item(db: Session, id: str): diff --git a/main.py b/main.py index dbc0409..6df84df 100644 --- a/main.py +++ b/main.py @@ -44,7 +44,7 @@ async def list_items(db: Session = Depends(get_db)): @app.put("/api/items/{id}") async def put_item(id: str, item: Item, db: Session = Depends(get_db)): - if crud.put_item(db, id, item): + if crud.put_item(db, id, item) == crud.PutItemResult.UPDATED: return Response(b"", status_code=204) return Response(b"", status_code=201) From 1bf4eb801f3a7f28f6f4104200a309eae2b73812 Mon Sep 17 00:00:00 2001 From: clonejo Date: Sun, 14 Jan 2024 20:23:08 +0100 Subject: [PATCH 03/12] Add last_updated field to be able to track outdated information / lost items --- main.py | 9 +++++++++ models.py | 3 ++- schemas.py | 3 ++- static/form.html | 3 +++ static/form.js | 6 ++++++ 5 files changed, 22 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 6df84df..23b9fd5 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,5 @@ import re +import time from fastapi import Depends, FastAPI, Response from fastapi.staticfiles import StaticFiles @@ -44,6 +45,8 @@ async def list_items(db: Session = Depends(get_db)): @app.put("/api/items/{id}") async def put_item(id: str, item: Item, db: Session = Depends(get_db)): + if item.last_updated is None: + item.last_updated = rough_timestamp() if crud.put_item(db, id, item) == crud.PutItemResult.UPDATED: return Response(b"", status_code=204) return Response(b"", status_code=201) @@ -54,4 +57,10 @@ async def delete_item(id: str, db: Session = Depends(get_db)): crud.delete_item(db, id) +def rough_timestamp() -> int: + """Provides an current timestamp with reduced resolution, to improve anonymity.""" + granularity = 2**20 # about 12 days + return int(time.time()) // granularity * granularity + + app.mount("/", StaticFiles(directory="static", html=True), name="static") diff --git a/models.py b/models.py index 7aa9b21..1df2c2b 100644 --- a/models.py +++ b/models.py @@ -1,4 +1,4 @@ -from sqlalchemy import Boolean, Column, ForeignKey, String, Text +from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, Text from database import Base @@ -15,3 +15,4 @@ class Item(Base): content = Column(Text) note = Column(Text) hidden = Column(Boolean, nullable=False) + last_updated = Column(Integer) diff --git a/schemas.py b/schemas.py index 218fe94..bf6f444 100644 --- a/schemas.py +++ b/schemas.py @@ -1,4 +1,4 @@ -from pydantic import BaseModel +from pydantic import BaseModel, Field class Item(BaseModel): @@ -11,6 +11,7 @@ class Item(BaseModel): content: str | None note: str | None hidden: bool + last_updated: int | None = Field(None) class Config: orm_mode = True diff --git a/static/form.html b/static/form.html index a5de3f2..d894279 100644 --- a/static/form.html +++ b/static/form.html @@ -43,6 +43,9 @@ + + + diff --git a/static/form.js b/static/form.js index 4ae4968..ec00949 100644 --- a/static/form.js +++ b/static/form.js @@ -39,6 +39,7 @@ function fillForm() { document.getElementById('content').value = item.content; document.getElementById('note').value = item.note; document.getElementById('hidden').checked = item.hidden; + document.getElementById('last_updated').value = formatTimestamp(item.last_updated); formCoordsToMap(); } @@ -172,3 +173,8 @@ function formCoordsToMap() { coordsToMap(coords_bl, coords_tr); } } + +function formatTimestamp(ts) { + const date = new Date(ts * 1000); + return date.toLocaleDateString(); +} From f3658a76ffcf22ba022ceb46c7583dc9bc0d1ee0 Mon Sep 17 00:00:00 2001 From: jomo Date: Thu, 1 Feb 2024 00:01:22 +0100 Subject: [PATCH 04/12] format database.py using black --- database.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/database.py b/database.py index ac9f8a6..44a3006 100644 --- a/database.py +++ b/database.py @@ -5,9 +5,7 @@ from sqlalchemy.orm import sessionmaker SQLALCHEMY_DATABASE_URL = "sqlite:///./inventory.sqlite3" engine = create_engine( - SQLALCHEMY_DATABASE_URL, - connect_args = {"check_same_thread": False}, - echo=True + SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}, echo=True ) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) From f1384b72f208b1623becb79f8ded7390cd524e74 Mon Sep 17 00:00:00 2001 From: jomo Date: Thu, 1 Feb 2024 00:01:55 +0100 Subject: [PATCH 05/12] orm_mode has been renamed to from_attributes --- schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas.py b/schemas.py index bf6f444..30f7545 100644 --- a/schemas.py +++ b/schemas.py @@ -14,4 +14,4 @@ class Item(BaseModel): last_updated: int | None = Field(None) class Config: - orm_mode = True + from_attributes = True From 621db6ad2c1064f8e9df015def60a9c9911fefe2 Mon Sep 17 00:00:00 2001 From: jomo Date: Thu, 1 Feb 2024 00:03:33 +0100 Subject: [PATCH 06/12] use current month for last_updated value --- main.py | 12 ++++++------ static/form.js | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/main.py b/main.py index 23b9fd5..138806b 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,6 @@ import re -import time +from calendar import timegm +from datetime import date from fastapi import Depends, FastAPI, Response from fastapi.staticfiles import StaticFiles @@ -46,7 +47,7 @@ async def list_items(db: Session = Depends(get_db)): @app.put("/api/items/{id}") async def put_item(id: str, item: Item, db: Session = Depends(get_db)): if item.last_updated is None: - item.last_updated = rough_timestamp() + item.last_updated = month_timestamp() if crud.put_item(db, id, item) == crud.PutItemResult.UPDATED: return Response(b"", status_code=204) return Response(b"", status_code=201) @@ -57,10 +58,9 @@ async def delete_item(id: str, db: Session = Depends(get_db)): crud.delete_item(db, id) -def rough_timestamp() -> int: - """Provides an current timestamp with reduced resolution, to improve anonymity.""" - granularity = 2**20 # about 12 days - return int(time.time()) // granularity * granularity +def month_timestamp() -> int: + """Provides the timestamp of the current month's beginning (for improved privacy)""" + return timegm(date.today().replace(day=1).timetuple()) app.mount("/", StaticFiles(directory="static", html=True), name="static") diff --git a/static/form.js b/static/form.js index ec00949..b69f358 100644 --- a/static/form.js +++ b/static/form.js @@ -176,5 +176,6 @@ function formCoordsToMap() { function formatTimestamp(ts) { const date = new Date(ts * 1000); - return date.toLocaleDateString(); + // using Swedish format as a hack to get an iso formatted date + return date.toLocaleDateString("sv", {timeZone: "UTC"}).replace(/\-\d+$/,'') } From 6d01523d8d91a7025129bc98c6cd2c4e65767cb4 Mon Sep 17 00:00:00 2001 From: jomo Date: Thu, 1 Feb 2024 01:04:20 +0100 Subject: [PATCH 07/12] update python version and modules --- Dockerfile | 2 +- main.py | 2 +- requirements.txt | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 228fbb6..9cfa1aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-alpine +FROM python:3.12-alpine3.19 WORKDIR /app diff --git a/main.py b/main.py index 138806b..cc4092d 100644 --- a/main.py +++ b/main.py @@ -37,7 +37,7 @@ def _set_sqlite_pragma(conn, _): async def list_items(db: Session = Depends(get_db)): # natural sort by id natsort = lambda item: [ - int(t) if t.isdigit() else t.lower() for t in re.split("(\d+)", item.id) + int(t) if t.isdigit() else t.lower() for t in re.split(r"(\d+)", item.id) ] items = crud.get_items(db) items = sorted(items, key=natsort) diff --git a/requirements.txt b/requirements.txt index 54b2b3d..92fc06b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -fastapi>=0.79.0 -sqlalchemy>=1.4.39 -uvicorn>=0.17.6 +fastapi>=0.109.0 +sqlalchemy>=2.0.25 +uvicorn>=0.27.0.post1 From 2b7342b90284c7d9b6ede9a3fe56ad07d83fb913 Mon Sep 17 00:00:00 2001 From: jomo Date: Thu, 1 Feb 2024 01:04:49 +0100 Subject: [PATCH 08/12] update openapi docs for PUT request --- main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index cc4092d..eb66394 100644 --- a/main.py +++ b/main.py @@ -44,7 +44,11 @@ async def list_items(db: Session = Depends(get_db)): return {i.id: i for i in items} -@app.put("/api/items/{id}") +@app.put( + "/api/items/{id}", + status_code=201, + responses={201: {"description": "created"}, 204: {"description": "updated"}}, +) async def put_item(id: str, item: Item, db: Session = Depends(get_db)): if item.last_updated is None: item.last_updated = month_timestamp() From ea0538f90ca7575d6fbc9bc640537ee4ae54bef4 Mon Sep 17 00:00:00 2001 From: jomo Date: Thu, 1 Feb 2024 01:42:00 +0100 Subject: [PATCH 09/12] ignore all dotfiles in .dockerignore --- .dockerignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 0fdd12c..3db5969 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,4 @@ -/.dockerignore -/.gitignore +.* /inventory.sqlite3 /README.md *.pyc From 8f5a82fb418df5fca5d926a18b9bb6ecc91ece34 Mon Sep 17 00:00:00 2001 From: clonejo Date: Sat, 6 Jan 2024 06:28:38 +0100 Subject: [PATCH 10/12] add client-side python script to create items in bulk --- contrib/bulk_create.py | 47 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 contrib/bulk_create.py diff --git a/contrib/bulk_create.py b/contrib/bulk_create.py new file mode 100755 index 0000000..8ed6aed --- /dev/null +++ b/contrib/bulk_create.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +import sys + +import requests + +BASE = "https://in.ccc.ac/api" +COUNT = 0 +START_ID = 1 + + +session = requests.Session() +# session.auth = (":)", ":)") + +res = session.get(f"{BASE}/items") +res.raise_for_status() +ids = res.json().keys() +kiste_ids = {int(i.lstrip("K")) for i in ids if i.startswith("K") and i != "K"} +print("existing ids:", repr(kiste_ids), file=sys.stderr) + +current_id = START_ID +new_ids = [] +while len(new_ids) < COUNT: + if current_id not in kiste_ids: + new_ids.append(current_id) + current_id += 1 + +print("created:", file=sys.stderr) +if len(new_ids) == 0: + print("(none)", file=sys.stderr) + + +for i in new_ids: + j = { + "id": f"K{i}", + "is_in": None, + "coords_bl": None, + "coords_tr": None, + "type": "Kiste", + "name": None, + "content": None, + "note": "Bitte Name, Ort, Inhalt eintragen", + "hidden": False, + } + res = session.put(f"{BASE}/items/{j['id']}", json=j) + res.raise_for_status() + print(f"{j['id']},https://in.ccc.ac/form.html?id={j['id']}") From ec64e818291059b04d7577d729275351edfe3db9 Mon Sep 17 00:00:00 2001 From: jomo Date: Thu, 1 Feb 2024 01:48:51 +0100 Subject: [PATCH 11/12] add contrib/ to .dockerignore --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 3db5969..4bc1268 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,3 +3,4 @@ /README.md *.pyc Dockerfile +/contrib/ From 703a210f1ef92e0449245472b3b8d53cb7721dff Mon Sep 17 00:00:00 2001 From: jomo Date: Thu, 1 Feb 2024 02:03:02 +0100 Subject: [PATCH 12/12] use BASE var in bulk_create output --- contrib/bulk_create.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/bulk_create.py b/contrib/bulk_create.py index 8ed6aed..3d7fbf5 100755 --- a/contrib/bulk_create.py +++ b/contrib/bulk_create.py @@ -4,7 +4,7 @@ import sys import requests -BASE = "https://in.ccc.ac/api" +BASE = "https://in.ccc.ac" COUNT = 0 START_ID = 1 @@ -12,7 +12,7 @@ START_ID = 1 session = requests.Session() # session.auth = (":)", ":)") -res = session.get(f"{BASE}/items") +res = session.get(f"{BASE}/api/items") res.raise_for_status() ids = res.json().keys() kiste_ids = {int(i.lstrip("K")) for i in ids if i.startswith("K") and i != "K"} @@ -42,6 +42,6 @@ for i in new_ids: "note": "Bitte Name, Ort, Inhalt eintragen", "hidden": False, } - res = session.put(f"{BASE}/items/{j['id']}", json=j) + res = session.put(f"{BASE}/api/items/{j['id']}", json=j) res.raise_for_status() - print(f"{j['id']},https://in.ccc.ac/form.html?id={j['id']}") + print(f"{j['id']},{BASE}/form.html?id={j['id']}")