From 9d8192dff6b3a1c6576742faebfa14fb1c1f407c Mon Sep 17 00:00:00 2001 From: Gavan Fantom Date: Thu, 2 Dec 2021 12:34:36 +0000 Subject: [PATCH] Add jlc-chart app --- config.py | 20 ++++++ jlcchart/__init__.py | 43 ++++++++++++ jlcchart/bom.py | 106 ++++++++++++++++++++++++++++++ jlcchart/components.py | 42 ++++++++++++ jlcchart/db.py | 39 +++++++++++ jlcchart/fetcher.py | 76 +++++++++++++++++++++ jlcchart/schema.sql | 13 ++++ jlcchart/templates/bom.html | 97 +++++++++++++++++++++++++++ jlcchart/templates/bom_form.html | 25 +++++++ jlcchart/templates/component.html | 65 ++++++++++++++++++ jlcchart/util.py | 8 +++ wsgi.py | 4 ++ 12 files changed, 538 insertions(+) create mode 100644 config.py create mode 100644 jlcchart/__init__.py create mode 100644 jlcchart/bom.py create mode 100644 jlcchart/components.py create mode 100644 jlcchart/db.py create mode 100644 jlcchart/fetcher.py create mode 100644 jlcchart/schema.sql create mode 100644 jlcchart/templates/bom.html create mode 100644 jlcchart/templates/bom_form.html create mode 100644 jlcchart/templates/component.html create mode 100644 jlcchart/util.py create mode 100644 wsgi.py diff --git a/config.py b/config.py new file mode 100644 index 0000000..cc3c248 --- /dev/null +++ b/config.py @@ -0,0 +1,20 @@ +"""Flask configuration.""" + +from os import environ, path +from dotenv import load_dotenv +import secrets + +basedir = path.abspath(path.dirname(__file__)) +dotfile = path.join(basedir, '.env') + +# Let's make this as easy as possible to deploy, shall we? +if not os.path.exists(dotfile): + with open(dotfile, 'w') as f: + f.write("SECRET_KEY = '{}'".format(secrets.token_urlsafe(16))) + +load_dotenv(dotfile) + +# For development, you know what to do +#FLASK_ENV = 'development' +FLASK_ENV = 'production' +SECRET_KEY = environ.get('SECRET_KEY') diff --git a/jlcchart/__init__.py b/jlcchart/__init__.py new file mode 100644 index 0000000..f78c6ac --- /dev/null +++ b/jlcchart/__init__.py @@ -0,0 +1,43 @@ +import os + +from flask import Flask, redirect, url_for + +from .util import ListConverter + +def create_app(test_config=None): + app = Flask(__name__, instance_relative_config=True) + app.config.from_mapping( + SECRET_KEY='dev', + DATABASE=os.path.join(app.instance_path, 'components.db'), + ) + + if test_config is None: + app.config.from_pyfile('config.py', silent=True) + else: + app.config.from_mapping(test_config) + + try: + os.makedirs(app.instance_path) + except OSError: + pass + + app.url_map.converters['list'] = ListConverter + + @app.route('/') + def index(): + return redirect(url_for('bom.bom')) + + from . import db + db.init_app(app) + + from . import fetcher + fetcher.init_app(app) + + from . import components + app.register_blueprint(components.bp) + + from . import bom + app.register_blueprint(bom.bp) + + return app + diff --git a/jlcchart/bom.py b/jlcchart/bom.py new file mode 100644 index 0000000..4502f82 --- /dev/null +++ b/jlcchart/bom.py @@ -0,0 +1,106 @@ +from flask import ( + Blueprint, flash, g, redirect, render_template, request, url_for, jsonify +) + +from werkzeug.exceptions import abort + +import json +import csv +import re +import io + +from .db import get_db +from .fetcher import get_fetcher + +bp = Blueprint('bom', __name__, url_prefix='/bom') + +def match(x): + m = re.search("C\d+", x) + if m: + return m.group() + else: + return None + +@bp.route('/add', methods=(['POST'])) +def add(): + db = get_db() + f = get_fetcher() + + if request.method == 'POST': + component = request.json['code'] + + row = db.execute("""SELECT id FROM fetchlist WHERE code = ?""", (component,)).fetchone() + if row is not None: + return jsonify(id=row[0]) + + j = f.fetch(component, base_url='https://cart.jlcpcb.com/shoppingCart/smtGood/getComponentDetail?{}', query='componentCode={}') + if j: + try: + j = json.loads(j) + lcsc_id = j['data']['lcscComponentId'] + except: + abort(404, "Component {} not found".format(component)) + db.execute("""INSERT INTO 'fetchlist' ('id', 'code') VALUES(?, ?);""", (lcsc_id, component)) + db.commit() + return jsonify(id=lcsc_id) + else: + abort(404, "Component {} not found".format(component)) + +@bp.route('/', methods=('GET', 'POST')) +def bom(): + db = get_db() + + bom = [] + if request.method == 'POST': + stream = io.StringIO(request.form['bom']) + r = csv.reader(stream) + for row in r: + bom.append(row) + maxlen = max(len(i) for i in bom) + for row in bom: + if len(row) < maxlen: + row.extend('' for _ in range(maxlen - len(row))) + if len(bom) < 1: + abort(400, "No BOM supplied") + has_header = True + for val in bom[0]: + if match(val): + has_header = False + if has_header: + header = bom[0] + bom = bom[1:] + else: + header = ['' for _ in range(maxlen)] + bom_col = list(zip(*bom)) + count = [] + for column in bom_col: + total = 0 + for x in column: + if match(x): + total += 1 + count.append(total) + componentcol = None + for i, val in enumerate(count): + if val == max(count): + componentcol = i + break + + if max(count) < 1: + abort(400, "BOM must contain at least one component") + components = [match(x) for x in bom_col[componentcol]] + + ids = [] + for component in components: + row = db.execute('SELECT id from fetchlist WHERE code=?', (component,)).fetchone() + if row: + ids.append(row[0]) + else: + ids.append('') + + header += ['JLC code', 'JLC ID'] + table_col = bom_col + [components] + [ids] + table = list(zip(*table_col)) + + return render_template('bom.html', headings=header, table=table) + return render_template('bom_form.html') + diff --git a/jlcchart/components.py b/jlcchart/components.py new file mode 100644 index 0000000..014593d --- /dev/null +++ b/jlcchart/components.py @@ -0,0 +1,42 @@ +from flask import ( + Blueprint, flash, g, redirect, render_template, request, url_for +) + +from werkzeug.exceptions import abort + +import json + +from .db import get_db + +bp = Blueprint('components', __name__, url_prefix='/components') + + +@bp.route('') +def components(components): + db = get_db() + results = [] + for component in components: + rows = db.execute("""SELECT timestamp, json_extract(data, '$.data.stockCount') AS stock FROM components WHERE id=? ORDER BY timestamp;""", (component,)).fetchall() +# if not rows: +# abort(404, f"Component {component} doesn't exist.") + + data = [] + for row in rows: + data.append({'timestamp' : row['timestamp'], 'value' : row['stock']}) + + row = db.execute("""SELECT timestamp, data FROM components WHERE id=? ORDER BY timestamp DESC LIMIT 1;""", (component,)).fetchone() + + #if row is None: + # abort(404, f"Error retrieving component {component}.") + + try: + details = json.loads(row['data']) + code = details['data']['componentCode'] + name = details['data']['componentName'] + except: + code = component + name = "Not Found" + results.append({'data': data, 'id': component, 'code': code, 'name': name}) + + return render_template('component.html', components=results) + diff --git a/jlcchart/db.py b/jlcchart/db.py new file mode 100644 index 0000000..023a059 --- /dev/null +++ b/jlcchart/db.py @@ -0,0 +1,39 @@ +import sqlite3 + +import click +from flask import current_app, g +from flask.cli import with_appcontext + +def get_db(): + if 'db' not in g: + g.db = sqlite3.connect( + current_app.config['DATABASE'], + detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES + ) + g.db.row_factory = sqlite3.Row + + return g.db + +def close_db(e=None): + db = g.pop('db', None) + + if db is not None: + db.close() + +def init_db(): + db = get_db() + + with current_app.open_resource('schema.sql') as f: + db.executescript(f.read().decode('utf8')) + +def init_app(app): + app.teardown_appcontext(close_db) + app.cli.add_command(init_db_command) + +@click.command('init-db') +@with_appcontext +def init_db_command(): + """Clear the existing data and create new tables.""" + init_db() + click.echo("Initialised the database.") + diff --git a/jlcchart/fetcher.py b/jlcchart/fetcher.py new file mode 100644 index 0000000..b6b5b5c --- /dev/null +++ b/jlcchart/fetcher.py @@ -0,0 +1,76 @@ +import sqlite3 + +import click +from flask import current_app, g +from flask.cli import with_appcontext + +import requests +import datetime +import time +import json + +from .db import get_db + +class fetcher: + def __init__(self, base_url='https://jlcpcb.com/shoppingCart/smtGood/getComponentDetail?{}', query='componentLcscId={}', period=1, retries=5, retry_delay=30): + self.base_url = base_url + self.last_use = 0 + self.period = period + self.retries = retries + self.retry_delay = retry_delay + self.query = query + + def fetch(self, component, base_url=None, query=None): + if base_url is None: + base_url = self.base_url + if query is None: + query = self.query + trycount = self.retries + while True: + now = time.monotonic() + while now - self.last_use < self.period: + time.sleep(now - self.last_use) + now = time.monotonic() + self.last_use = now + #print("Querying {}".format(base_url.format(query.format(component)))) + r = requests.get(base_url.format(query.format(component))) + #print("Got status {}".format(r.status_code)) + if r.status_code == 200: + return r.text + if trycount > self.retries: + return None + trycount += 1 + time.sleep(self.retry_delay) + +def get_fetcher(): + if 'f' not in g: + g.f = fetcher() + + return g.f + +def init_app(app): + app.cli.add_command(poll_command) + +@click.command('poll') +@with_appcontext +def poll_command(): + """Fetch data from JLCPCB.""" + db = get_db() + f = get_fetcher() + + components = db.execute("""select id from fetchlist;""").fetchall() + + sqlite_insert_with_param = """INSERT INTO 'components' ('id', 'timestamp', 'data') VALUES(?, ?, ?);""" + + for (component,) in components: + j = f.fetch(component) + timestamp = datetime.datetime.now() + if j: + data = (component, timestamp, j) + db.execute(sqlite_insert_with_param, data) + db.commit() + else: + click.echo("Failed to fetch component {}".format(component)) + + click.echo("Polling complete") + diff --git a/jlcchart/schema.sql b/jlcchart/schema.sql new file mode 100644 index 0000000..7e28684 --- /dev/null +++ b/jlcchart/schema.sql @@ -0,0 +1,13 @@ +DROP TABLE IF EXISTS components; +DROP TABLE IF EXISTS fetchlist; + +CREATE TABLE IF NOT EXISTS components ( + id INTEGER, + [timestamp] timestamp, + data TEXT +); + +CREATE TABLE IF NOT EXISTS fetchlist ( + id INTEGER, + code TEXT +); diff --git a/jlcchart/templates/bom.html b/jlcchart/templates/bom.html new file mode 100644 index 0000000..1db9abf --- /dev/null +++ b/jlcchart/templates/bom.html @@ -0,0 +1,97 @@ + + + + + JLCPCB stock query: BOM + + + + + + + + {% for item in headings -%} + + {% endfor -%} + + + {% for row in table -%} + {% set rowloop = loop -%} + + {% for item in row -%} + {% if loop.last and item == '' -%} + + {% else -%} + + {% endif -%} + {% endfor -%} + + + {% endfor -%} +
{{ item }}View
{{ item }}
+

+

See the graphs!

+ + diff --git a/jlcchart/templates/bom_form.html b/jlcchart/templates/bom_form.html new file mode 100644 index 0000000..472be95 --- /dev/null +++ b/jlcchart/templates/bom_form.html @@ -0,0 +1,25 @@ + + + + + JLCPCB stock query + + + + +
+

Paste your BOM here

+ +

+

+ +
+
+ + diff --git a/jlcchart/templates/component.html b/jlcchart/templates/component.html new file mode 100644 index 0000000..fe8f1b9 --- /dev/null +++ b/jlcchart/templates/component.html @@ -0,0 +1,65 @@ + + + + + JLCPCB stock graphs + + + + + + + + + {% for component in components %} + + {% endfor %} + + + + diff --git a/jlcchart/util.py b/jlcchart/util.py new file mode 100644 index 0000000..f644b14 --- /dev/null +++ b/jlcchart/util.py @@ -0,0 +1,8 @@ +from werkzeug.routing import BaseConverter + +class ListConverter(BaseConverter): + def to_python(self, value): + return value.split('+') + + def to_url(self, values): + return '+'.join(BaseConverter.to_url(value) for value in values) diff --git a/wsgi.py b/wsgi.py new file mode 100644 index 0000000..5f3fc78 --- /dev/null +++ b/wsgi.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python3 + +from jlcchart import create_app +application = create_app()