176 lines
8.8 KiB
Python
176 lines
8.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Reconcile a datasheet-extractor run against the DFS + Parameters Gitea repos.
|
|
|
|
Only DFS and Parameters are MPN-indexed. DFS has one MPN_make_typeid/ folder per part;
|
|
Parameters holds a SINGLE master workbook (Components_Master.xlsx) whose type sheets each
|
|
have one row per part (column A). Skill_Assets holds the skill + templates (not per-part),
|
|
so it is never reconciled here - push it with push_to_gitea.sh.
|
|
|
|
Identity = the part tag <MPN>_<make>_<typeid> (fill_templates.part_tag). Same tag = same
|
|
part, so re-processing an MPN already in Gitea is a conflict.
|
|
|
|
CHECK (no writes/push):
|
|
python scripts/gitea_reconcile.py --parts parts.json --dfs-src <dfs_stage> \
|
|
--template assets/template/template.xlsx --report conflicts.json
|
|
APPLY + push:
|
|
python scripts/gitea_reconcile.py --parts parts.json --dfs-src <dfs_stage> \
|
|
--template assets/template/template.xlsx --decisions decisions.json --push
|
|
# or fully unattended: --on-conflict replace (or discard)
|
|
|
|
replace -> overwrite that MPN's DFS folder AND its row in the master sheet.
|
|
discard -> keep the copy already in Gitea; drop the new one.
|
|
New MPNs are always added; every other folder/row/sheet is preserved (the master is
|
|
merged, never wholesale-overwritten). Refuses to push if a conflict has no decision.
|
|
"""
|
|
import argparse, json, os, re, shutil, subprocess, sys, tempfile
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import openpyxl
|
|
import fill_templates as ft
|
|
|
|
def load_env(cfg):
|
|
env={}
|
|
if os.path.exists(cfg):
|
|
for line in open(cfg):
|
|
line=line.strip()
|
|
if not line or line.startswith("#") or "=" not in line: continue
|
|
k,v=line.split("=",1); env[k.strip()]=v.strip()
|
|
for k in ("GIT_HOST","GIT_USER","GIT_TOKEN","DFS_REPO","PARAMS_REPO"):
|
|
if os.environ.get(k): env[k]=os.environ[k]
|
|
return env
|
|
|
|
def repo_url(env, repo):
|
|
host=re.sub(r'^https?://','',env["GIT_HOST"]).rstrip('/')
|
|
cred=(env.get("GIT_USER","")+":" if env.get("GIT_USER") else "")+env["GIT_TOKEN"]
|
|
return f"https://{cred}@{host}/{repo}.git"
|
|
|
|
def clone(url, dest, token):
|
|
r=subprocess.run(["git","clone","--depth","1",url,dest], capture_output=True, text=True)
|
|
return r.returncode==0, (r.stderr or "").replace(token,"***")
|
|
|
|
def run_rows(parts): return [(ft.part_tag(p), p["type"], p.get("mpn",""), p) for p in parts]
|
|
def master_path(params_clone): return os.path.join(params_clone, ft.MASTER_NAME)
|
|
def _width(template, ctype):
|
|
h=ft.template_headers(template, ctype); return len(h) if h else 0
|
|
|
|
def scan_conflicts(parts, dfs_clone, params_clone, template):
|
|
conflicts=[]; tags_by_type={}
|
|
mp=master_path(params_clone)
|
|
wb=openpyxl.load_workbook(mp) if os.path.exists(mp) else None
|
|
for tag,ctype,mpn,_ in run_rows(parts):
|
|
in_dfs=os.path.isdir(os.path.join(dfs_clone,tag))
|
|
if ctype not in tags_by_type:
|
|
s=set(); w=_width(template,ctype)
|
|
if wb is not None and ctype in wb.sheetnames and w:
|
|
_,rows=ft.sheet_rows(wb[ctype], w); s=set(rows.keys())
|
|
tags_by_type[ctype]=s
|
|
in_params=tag in tags_by_type[ctype]
|
|
if in_dfs or in_params:
|
|
conflicts.append({"tag":tag,"type":ctype,"mpn":mpn,
|
|
"in_dfs":in_dfs,"in_params":in_params})
|
|
return conflicts
|
|
|
|
def apply_dfs(parts, dfs_src, dfs_clone, decisions):
|
|
for tag,_,_,_ in run_rows(parts):
|
|
src=os.path.join(dfs_src, tag)
|
|
if not os.path.isdir(src): continue
|
|
dst=os.path.join(dfs_clone, tag); dec=decisions.get(tag)
|
|
if os.path.isdir(dst):
|
|
if dec=="discard": print(f" DFS keep existing : {tag}")
|
|
elif dec=="replace":
|
|
shutil.rmtree(dst); shutil.copytree(src,dst); print(f" DFS replaced : {tag}")
|
|
else: print(f" DFS SKIP (undecided): {tag}")
|
|
else:
|
|
shutil.copytree(src,dst); print(f" DFS added : {tag}")
|
|
|
|
def apply_params(parts, params_clone, template, decisions, version):
|
|
"""Merge run rows into the master workbook, honouring decisions, and write it back.
|
|
Reads the full existing master so every other sheet/row is preserved; empty sheets stay
|
|
out (build_master keeps only sheets that have parts)."""
|
|
mp=master_path(params_clone)
|
|
complete={k:dict(v) for k,v in ft.read_all_rows(mp, template).items()} if os.path.exists(mp) else {}
|
|
for p in parts:
|
|
headers=ft.template_headers(template, p["type"])
|
|
if headers is None: print(f" ! no sheet for '{p['type']}' - skipping"); continue
|
|
tag,row=ft.part_to_row(p, headers); sheet=complete.setdefault(p["type"], {})
|
|
if tag in sheet:
|
|
dec=decisions.get(tag)
|
|
if dec=="discard": print(f" PARAMS keep row : {tag}")
|
|
elif dec=="replace": sheet[tag]=row; print(f" PARAMS replaced : {tag}")
|
|
else: print(f" PARAMS SKIP (undecided): {tag}")
|
|
else:
|
|
sheet[tag]=row; print(f" PARAMS added : {tag}")
|
|
ft.build_master(template, complete, version, params_clone)
|
|
|
|
def commit_push(clone_dir, msg, token):
|
|
for args in (["config","user.email","datasheet-bot@local"],
|
|
["config","user.name","datasheet-extractor"],
|
|
["checkout","-B","main"], ["add","-A"]):
|
|
subprocess.run(["git","-C",clone_dir]+args, capture_output=True)
|
|
if subprocess.run(["git","-C",clone_dir,"diff","--cached","--quiet"]).returncode==0:
|
|
print(f" nothing new for {os.path.basename(clone_dir)}"); return
|
|
subprocess.run(["git","-C",clone_dir,"commit","-m",msg], capture_output=True)
|
|
r=subprocess.run(["git","-C",clone_dir,"push","-u","origin","main"],
|
|
capture_output=True, text=True)
|
|
print(" "+((r.stdout+r.stderr).replace(token,"***").strip() or "pushed."))
|
|
|
|
def undecided(conflicts, decisions):
|
|
return [c["tag"] for c in conflicts if decisions.get(c["tag"]) not in ("replace","discard")]
|
|
|
|
def main():
|
|
ap=argparse.ArgumentParser()
|
|
ap.add_argument("--parts", required=True)
|
|
ap.add_argument("--dfs-src", required=True, help="staging dir of MPN_make_typeid/ folders")
|
|
ap.add_argument("--template", required=True)
|
|
ap.add_argument("--decisions", default=None, help="JSON map: tag -> replace|discard")
|
|
ap.add_argument("--report", default=None, help="write the conflict report here")
|
|
ap.add_argument("--push", action="store_true", help="apply decisions and push")
|
|
ap.add_argument("--on-conflict", choices=["ask","replace","discard"], default="ask",
|
|
help="what to do when an MPN already exists: ask the user (default), or "
|
|
"apply one policy to every conflict for an unattended run")
|
|
ap.add_argument("--config",
|
|
default=os.path.join(os.path.dirname(__file__),"..","config","gitea.env"))
|
|
a=ap.parse_args()
|
|
|
|
parts=json.load(open(a.parts))["parts"]
|
|
env=load_env(a.config)
|
|
for k in ("GIT_HOST","GIT_TOKEN","DFS_REPO","PARAMS_REPO"):
|
|
if not env.get(k): sys.exit(f"missing {k} in env/config")
|
|
token=env["GIT_TOKEN"]; version=ft.resolve_version(a.template, None)
|
|
decisions=json.load(open(a.decisions)) if a.decisions else {}
|
|
|
|
tmp=tempfile.mkdtemp(prefix="reconcile_")
|
|
dfs_clone=os.path.join(tmp,"dfs"); params_clone=os.path.join(tmp,"params")
|
|
ok,err=clone(repo_url(env,env["DFS_REPO"]), dfs_clone, token)
|
|
if not ok: sys.exit(f"clone DFS failed (host reachable / token scope?):\n{err[:400]}")
|
|
ok,err=clone(repo_url(env,env["PARAMS_REPO"]), params_clone, token)
|
|
if not ok: sys.exit(f"clone Parameters failed:\n{err[:400]}")
|
|
|
|
conflicts=scan_conflicts(parts, dfs_clone, params_clone, a.template)
|
|
report={"has_conflicts":bool(conflicts),"conflicts":conflicts}
|
|
if a.report: json.dump(report, open(a.report,"w"), indent=2)
|
|
print(json.dumps(report, indent=2))
|
|
|
|
if not a.push:
|
|
if conflicts:
|
|
print(f"\n{len(conflicts)} existing MPN(s) found — ask the user discard/replace, "
|
|
"then re-run with --decisions/--push (or set --on-conflict for unattended).")
|
|
else:
|
|
print("\nNo conflicts — re-run with --push to add everything (no prompt needed).")
|
|
return
|
|
|
|
if a.on_conflict in ("replace","discard"):
|
|
for c in conflicts: decisions.setdefault(c["tag"], a.on_conflict)
|
|
miss=undecided(conflicts, decisions)
|
|
if miss:
|
|
sys.exit("Refusing to push — need a discard/replace decision (or --on-conflict) for: "
|
|
+", ".join(miss))
|
|
print("Applying to DFS:"); apply_dfs(parts, a.dfs_src, dfs_clone, decisions)
|
|
print("Applying to Parameters:"); apply_params(parts, params_clone, a.template, decisions, version)
|
|
print("Pushing:")
|
|
commit_push(dfs_clone, "datasheet-extractor: reconcile DFS", token)
|
|
commit_push(params_clone, "datasheet-extractor: reconcile master parameters", token)
|
|
print("\nDone — DFS + master Parameters reconciled and pushed.")
|
|
|
|
if __name__=="__main__":
|
|
main()
|