Add jlc-chart app

jlcchart/__init__.py

@@ -0,0 +1,43 @@
+import os
+
+from flask import Flask, redirect, url_for
+
+from .util import ListConverter
+
+def create_app(test_config=None):
+    app = Flask(__name__, instance_relative_config=True)
+    app.config.from_mapping(
+        SECRET_KEY='dev',
+        DATABASE=os.path.join(app.instance_path, 'components.db'),
+    )
+
+    if test_config is None:
+        app.config.from_pyfile('config.py', silent=True)
+    else:
+        app.config.from_mapping(test_config)
+
+    try:
+        os.makedirs(app.instance_path)
+    except OSError:
+        pass
+
+    app.url_map.converters['list'] = ListConverter
+
+    @app.route('/')
+    def index():
+        return redirect(url_for('bom.bom'))
+
+    from . import db
+    db.init_app(app)
+
+    from . import fetcher
+    fetcher.init_app(app)
+
+    from . import components
+    app.register_blueprint(components.bp)
+
+    from . import bom
+    app.register_blueprint(bom.bp)
+
+    return app
+

jlcchart/bom.py

@@ -0,0 +1,106 @@
+from flask import (
+    Blueprint, flash, g, redirect, render_template, request, url_for, jsonify
+)
+
+from werkzeug.exceptions import abort
+
+import json
+import csv
+import re
+import io
+
+from .db import get_db
+from .fetcher import get_fetcher
+
+bp = Blueprint('bom', __name__, url_prefix='/bom')
+
+def match(x):
+    m = re.search("C\d+", x)
+    if m:
+        return m.group()
+    else:
+        return None
+
+@bp.route('/add', methods=(['POST']))
+def add():
+    db = get_db()
+    f = get_fetcher()
+
+    if request.method == 'POST':
+        component = request.json['code']
+
+        row = db.execute("""SELECT id FROM fetchlist WHERE code = ?""", (component,)).fetchone()
+        if row is not None:
+            return jsonify(id=row[0])
+
+        j = f.fetch(component, base_url='https://cart.jlcpcb.com/shoppingCart/smtGood/getComponentDetail?{}', query='componentCode={}')
+        if j:
+            try:
+                j = json.loads(j)
+                lcsc_id = j['data']['lcscComponentId']
+            except:
+                abort(404, "Component {} not found".format(component))
+            db.execute("""INSERT INTO 'fetchlist' ('id', 'code') VALUES(?, ?);""", (lcsc_id, component))
+            db.commit()
+            return jsonify(id=lcsc_id)
+        else:
+            abort(404, "Component {} not found".format(component))
+
+@bp.route('/', methods=('GET', 'POST'))
+def bom():
+    db = get_db()
+
+    bom = []
+    if request.method == 'POST':
+        stream = io.StringIO(request.form['bom'])
+        r = csv.reader(stream)
+        for row in r:
+            bom.append(row)
+        maxlen = max(len(i) for i in bom)
+        for row in bom:
+            if len(row) < maxlen:
+                row.extend('' for _ in range(maxlen - len(row)))
+        if len(bom) < 1:
+            abort(400, "No BOM supplied")
+        has_header = True
+        for val in bom[0]:
+            if match(val):
+                has_header = False
+        if has_header:
+            header = bom[0]
+            bom = bom[1:]
+        else:
+            header = ['' for _ in range(maxlen)]
+        bom_col = list(zip(*bom))
+        count = []
+        for column in bom_col:
+            total = 0
+            for x in column:
+                if match(x):
+                    total += 1
+            count.append(total)
+        componentcol = None
+        for i, val in enumerate(count):
+            if val == max(count):
+                componentcol = i
+                break
+
+        if max(count) < 1:
+            abort(400, "BOM must contain at least one component")
+        components = [match(x) for x in bom_col[componentcol]]
+
+        ids = []
+        for component in components:
+            row = db.execute('SELECT id from fetchlist WHERE code=?', (component,)).fetchone()
+            if row:
+                ids.append(row[0])
+            else:
+                ids.append('')
+
+        header += ['JLC code', 'JLC ID']
+        table_col = bom_col + [components] + [ids]
+        table = list(zip(*table_col))
+
+        return render_template('bom.html', headings=header, table=table)
+    return render_template('bom_form.html')
+

jlcchart/components.py

@@ -0,0 +1,42 @@
+from flask import (
+    Blueprint, flash, g, redirect, render_template, request, url_for
+)
+
+from werkzeug.exceptions import abort
+
+import json
+
+from .db import get_db
+
+bp = Blueprint('components', __name__, url_prefix='/components')
+
+
+@bp.route('<list:components>')
+def components(components):
+    db = get_db()
+    results = []
+    for component in components:
+        rows = db.execute("""SELECT timestamp, json_extract(data, '$.data.stockCount') AS stock FROM components WHERE id=? ORDER BY timestamp;""", (component,)).fetchall()
+#        if not rows:
+#            abort(404, f"Component {component} doesn't exist.")
+
+        data = []
+        for row in rows:
+            data.append({'timestamp' : row['timestamp'], 'value' : row['stock']})
+
+        row = db.execute("""SELECT timestamp, data FROM components WHERE id=? ORDER BY timestamp DESC LIMIT 1;""", (component,)).fetchone()
+
+        #if row is None:
+        #    abort(404, f"Error retrieving component {component}.")
+
+        try:
+            details = json.loads(row['data'])
+            code = details['data']['componentCode']
+            name = details['data']['componentName']
+        except:
+            code = component 
+            name = "Not Found"
+        results.append({'data': data, 'id': component, 'code': code, 'name': name})
+
+    return render_template('component.html', components=results)
+

jlcchart/db.py

@@ -0,0 +1,39 @@
+import sqlite3
+
+import click
+from flask import current_app, g
+from flask.cli import with_appcontext
+
+def get_db():
+    if 'db' not in g:
+        g.db = sqlite3.connect(
+            current_app.config['DATABASE'],
+            detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES
+        )
+        g.db.row_factory = sqlite3.Row
+
+    return g.db
+
+def close_db(e=None):
+    db = g.pop('db', None)
+
+    if db is not None:
+        db.close()
+
+def init_db():
+    db = get_db()
+
+    with current_app.open_resource('schema.sql') as f:
+        db.executescript(f.read().decode('utf8'))
+
+def init_app(app):
+    app.teardown_appcontext(close_db)
+    app.cli.add_command(init_db_command)
+
+@click.command('init-db')
+@with_appcontext
+def init_db_command():
+    """Clear the existing data and create new tables."""
+    init_db()
+    click.echo("Initialised the database.")
+

jlcchart/fetcher.py

@@ -0,0 +1,76 @@
+import sqlite3
+
+import click
+from flask import current_app, g
+from flask.cli import with_appcontext
+
+import requests
+import datetime
+import time
+import json
+
+from .db import get_db
+
+class fetcher:
+    def __init__(self, base_url='https://jlcpcb.com/shoppingCart/smtGood/getComponentDetail?{}', query='componentLcscId={}', period=1, retries=5, retry_delay=30):
+        self.base_url = base_url
+        self.last_use = 0
+        self.period = period
+        self.retries = retries
+        self.retry_delay = retry_delay
+        self.query = query
+
+    def fetch(self, component, base_url=None, query=None):
+        if base_url is None:
+            base_url = self.base_url
+        if query is None:
+            query = self.query
+        trycount = self.retries
+        while True:
+            now = time.monotonic()
+            while now - self.last_use < self.period:
+                time.sleep(now - self.last_use)
+                now = time.monotonic()
+            self.last_use = now
+            #print("Querying {}".format(base_url.format(query.format(component))))
+            r = requests.get(base_url.format(query.format(component)))
+            #print("Got status {}".format(r.status_code))
+            if r.status_code == 200:
+                return r.text
+            if trycount > self.retries:
+                return None
+            trycount += 1
+            time.sleep(self.retry_delay)
+
+def get_fetcher():
+    if 'f' not in g:
+        g.f = fetcher()
+
+    return g.f
+
+def init_app(app):
+    app.cli.add_command(poll_command)
+
+@click.command('poll')
+@with_appcontext
+def poll_command():
+    """Fetch data from JLCPCB."""
+    db = get_db()
+    f = get_fetcher()
+
+    components = db.execute("""select id from fetchlist;""").fetchall()
+
+    sqlite_insert_with_param = """INSERT INTO 'components' ('id', 'timestamp', 'data') VALUES(?, ?, ?);"""
+
+    for (component,) in components:
+        j = f.fetch(component)
+        timestamp = datetime.datetime.now()
+        if j:
+            data = (component, timestamp, j)
+            db.execute(sqlite_insert_with_param, data)
+            db.commit()
+        else:
+            click.echo("Failed to fetch component {}".format(component))
+
+    click.echo("Polling complete")
+

jlcchart/schema.sql

@@ -0,0 +1,13 @@
+DROP TABLE IF EXISTS components;
+DROP TABLE IF EXISTS fetchlist;
+
+CREATE TABLE IF NOT EXISTS components (
+    id INTEGER,
+    [timestamp] timestamp,
+    data TEXT
+);
+
+CREATE TABLE IF NOT EXISTS fetchlist (
+    id INTEGER,
+    code TEXT
+);

jlcchart/templates/bom.html

@@ -0,0 +1,97 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <title>JLCPCB stock query: BOM</title>
+  <style>
+table, th, td {
+    border: 1px solid #ddd;
+    border-collapse: collapse;
+    padding: 8px;
+}
+th {
+    padding-top: 12px;
+    padding-bottom: 12px;
+    background-color: #04AA6D;
+    color: white;
+}
+tr:nth-child(even) {background-color: #f2f2f2;}
+tr:hover {background-color: #ddd;}
+.error {
+    background-color: #ff0000;
+    color: white;
+}
+  </style>
+  <script>
+      function add_to_db(code, id) {
+          fetch('{{ url_for('bom.add') }}', {
+              method:'POST',
+              headers: {
+                  'Content-Type': 'application/json'
+              },
+              body: JSON.stringify({
+                  'code': code
+              })
+          })
+              .then(response => {
+                  if (!response.ok) {
+                      return Promise.reject("Request failed")
+                  } else {
+                      return response.json()
+                  }
+              })
+              .then(data => {
+                  if (data.id) {
+                      document.getElementById(id).innerHTML = data.id;
+                      update_link();
+                  }
+              })
+              .catch((error) => {
+                  document.getElementById(id).innerHTML = '<div class="error">FAIL</div>';
+                  console.error('Error', error);
+              });
+      }
+      function update_link() {
+          var rows = {{table|length}};
+          var id_col = {{headings|length}};
+          var idlist = [];
+          for (let i = 1; i <= rows; i++) {
+              if (document.getElementById('select_' + i).checked) {
+                  var id = document.getElementById('td_' + i + '_' + id_col).innerHTML;
+                  if (!isNaN(id)) {
+                      idlist.push(id)
+                  }
+              }
+          }
+          var url = '{{ url_for('components.components', components='') }}' + idlist.join('+');
+          document.getElementById('graphlink').href = url;
+      }
+  </script>
+</head>
+
+<body>
+    <table> 
+    <tr>
+    {% for item in headings -%}
+      <th>{{ item }}</th>
+    {% endfor -%}
+    <th>View</th>
+    </tr>
+    {% for row in table -%}
+      {% set rowloop = loop -%}
+      <tr>
+      {% for item in row -%}
+        {% if loop.last and item == '' -%}
+        <td id="td_{{rowloop.index}}_{{loop.index}}"><button onclick="add_to_db('{{loop.previtem}}', 'td_{{rowloop.index}}_{{loop.index}}')">Fetch data</button></td>
+        {% else -%}
+        <td id="td_{{rowloop.index}}_{{loop.index}}">{{ item }}</td>
+        {% endif -%}
+      {% endfor -%}
+      <td><input type="checkbox" id="select_{{rowloop.index}}" checked=checked onclick="update_link()"></td>
+      </tr>
+    {% endfor -%}
+    </table>
+    <p/>
+    <center><h1><a id='graphlink' href='{{ url_for('components.components', components='') }}{{table | map('last') | select('number') | join('+') }}'>See the graphs!</a></h1></center>
+</body>
+</html>

jlcchart/templates/bom_form.html

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <title>JLCPCB stock query</title>
+  <style>
+textarea
+{
+    width: 98%;
+    height: auto;
+}
+  </style>
+</head>
+
+<body>
+    <div align="center">
+        <h1>Paste your BOM here</h1>
+        <textarea name="bom" rows="50" form="bomform" placeholder="Paste your CSV file here"></textarea>
+        <p/>
+        <form action="{{ url_for('bom.bom') }}" id="bomform" method="POST">
+            <input type="submit" value="Submit BOM">
+        </form>
+    </div>
+</body>
+</html>

jlcchart/templates/component.html

@@ -0,0 +1,65 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <title>JLCPCB stock graphs</title>
+  <script src='https://cdnjs.cloudflare.com/ajax/libs/Chart.js/3.6.0/chart.min.js'></script>
+  <script src='https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.29.1/moment.min.js'></script>
+  <script src='https://cdnjs.cloudflare.com/ajax/libs/chartjs-adapter-moment/1.0.0/chartjs-adapter-moment.min.js'></script>
+</head>
+
+<body>
+<!---  <center> -->
+
+   {% for component in components %}
+    <canvas id="chart_{{component.id}}" width="600" height="400" style="display: inline;"></canvas>
+   {% endfor %}
+    <script>
+     {% for component in components %}
+      var chart_{{component.id}} = document.getElementById("chart_{{component.id}}").getContext("2d");
+      var LineChart_{{component.id}} = new Chart(chart_{{component.id}}, {
+          type: 'line',
+          data: {
+              datasets : [{
+                  label: '# in stock',
+                  data : [
+                  {% for item in component.data %}
+                      { x: '{{ item.timestamp }}', y: '{{item.value}}' },
+                  {% endfor %}]
+              }]
+          },
+          options: {
+              scales: {
+                  xAxis: {
+                      type: 'time',
+                      time: {
+                          unit: 'day'
+                      }
+                  },
+                  y: {
+                      beginAtZero: true
+                  }
+              },
+              responsive: false,
+              backgroundColor: "rgba(57, 204, 96, 0.7)",
+              borderColor: "rgba(57, 204, 96, 1)",
+              elements: {
+                  point: {
+                      pointBackgroundColor: "rgba(36, 128, 61, 0.7)",
+                      pointBorderColor: "rgba(36, 128, 61, 1)",
+                  }
+              },
+              plugins: {
+                  title: {
+                      display: true,
+                      text: ' {{ component.code }} : {{ component.name }} ',
+                      position: 'bottom'
+                  }
+              }
+          }
+      });
+     {% endfor %}
+    </script>
+<!---  </center> -->
+</body>
+</html>

jlcchart/util.py

@@ -0,0 +1,8 @@
+from werkzeug.routing import BaseConverter
+
+class ListConverter(BaseConverter):
+    def to_python(self, value):
+        return value.split('+')
+
+    def to_url(self, values):
+        return '+'.join(BaseConverter.to_url(value) for value in values)