jlc-chart/jlcchart/fetcher.py

import click
from flask import current_app, g
from flask.cli import with_appcontext

import requests
import datetime
import time
import json

from .db import get_db

class fetcher:
    def __init__(self, base_url='https://jlcpcb.com/shoppingCart/smtGood/getComponentDetail?{}', query='componentLcscId={}', period=1, retries=5, retry_delay=30):
        self.base_url = base_url
        self.last_use = 0
        self.period = period
        self.retries = retries
        self.retry_delay = retry_delay
        self.query = query

    def fetch(self, component, base_url=None, query=None):
        if base_url is None:
            base_url = self.base_url
        if query is None:
            query = self.query
        trycount = self.retries
        while True:
            now = time.monotonic()
            while now - self.last_use < self.period:
                time.sleep(now - self.last_use)
                now = time.monotonic()
            self.last_use = now
            #print("Querying {}".format(base_url.format(query.format(component))))
            r = requests.get(base_url.format(query.format(component)))
            #print("Got status {}".format(r.status_code))
            if r.status_code == 200:
                return r.text
            if trycount > self.retries:
                return None
            trycount += 1
            time.sleep(self.retry_delay)

def get_fetcher():
    if 'f' not in g:
        g.f = fetcher()

    return g.f

def init_app(app):
    app.cli.add_command(poll_command)

@click.command('poll')
@with_appcontext
def poll_command():
    """Fetch data from JLCPCB."""
    db = get_db()
    f = get_fetcher()

    components = db.execute("""select id from fetchlist;""").fetchall()

    sqlite_insert_with_param = """INSERT INTO 'components' ('id', 'timestamp', 'data') VALUES(?, ?, ?);"""

    for (component,) in components:
        j = f.fetch(component)
        timestamp = datetime.datetime.now()
        if j:
            data = (component, timestamp, j)
            db.execute(sqlite_insert_with_param, data)
            db.commit()
        else:
            click.echo("Failed to fetch component {}".format(component))

    click.echo("Polling complete")