mirror of
https://github.com/ilri/csv-metadata-quality-web.git
synced 2024-11-26 08:28:19 +01:00
Major refactor
Re-work upload and file processing so they are in the same Python function. Now I will start exposing other command line options in the form, like unsafe fixes, excluding fields, etc. Now I see tha t it is easier to save the POSTed file and process it in the same function so I don't have to pass around the other POSTed form val ues as URL query parameters. Now, as a result of changing the flow above, I also had to make a change to the way I show the results page. Instead of processing the file and returning the rendered results to the user directly, I process the file, save the rendered results to /tmp, and return a redirect to the user to the results page.
This commit is contained in:
parent
adc2d06094
commit
198acdb1a7
GPG Key ID:
0FB860CC9C45B1B9
79
main.py
79
main.py
@ -29,7 +29,7 @@ def index():
|
||||
|
||||
|
||||
@app.route("/", methods=["POST"])
|
||||
def upload_file():
|
||||
def process():
|
||||
uploaded_file = request.files["file"]
|
||||
filename = secure_filename(uploaded_file.filename)
|
||||
|
||||
@ -43,42 +43,59 @@ def upload_file():
|
||||
# generate a base64 representation of the filename to use as a slug
|
||||
base64name = b64encode(filename.encode("ascii"))
|
||||
|
||||
return redirect(url_for("process_file", base64slug=base64name))
|
||||
# do we need to use secure_filename again here?
|
||||
input_file = os.path.join(app.config["UPLOAD_PATH"], filename)
|
||||
# write output file with the same name as the input file plus "-cleaned"
|
||||
output_file = os.path.join(
|
||||
app.config["UPLOAD_PATH"], os.path.splitext(filename)[0] + "-cleaned.csv"
|
||||
)
|
||||
|
||||
args = ["-i", input_file, "-o", output_file]
|
||||
|
||||
if "unsafe" in request.form:
|
||||
args.append("-u")
|
||||
|
||||
# run subprocess and capture output as UTF-8 so we get a string instead of
|
||||
# bytes for ansi2html
|
||||
results = subprocess.run(
|
||||
["csv-metadata-quality"] + args,
|
||||
capture_output=True,
|
||||
encoding="UTF-8",
|
||||
)
|
||||
# convert the output to HTML using ansi2html
|
||||
conv = Ansi2HTMLConverter()
|
||||
stdout_html = conv.convert(results.stdout)
|
||||
|
||||
# render the results to HTML so we can save them for later and allowing
|
||||
# the user to share the results page without posting the file again. We
|
||||
# decode base64name before sending it to convert it from bytes to str.
|
||||
results_html = render_template(
|
||||
"result.html",
|
||||
cli_version=cli_version,
|
||||
filename=filename,
|
||||
stdout=stdout_html,
|
||||
base64name=base64name.decode("ascii"),
|
||||
)
|
||||
# save results to a file so it's easy to have a saved results page when
|
||||
# we don't know the options a user used to POST the form.
|
||||
results_html_file = os.path.join(
|
||||
app.config["UPLOAD_PATH"], base64name.decode("ascii")
|
||||
)
|
||||
with open(results_html_file, "w") as fh:
|
||||
fh.write(results_html)
|
||||
|
||||
return redirect(url_for("results", base64slug=base64name))
|
||||
|
||||
return "No file selected"
|
||||
|
||||
|
||||
@app.route("/result/<base64slug>")
|
||||
def process_file(base64slug):
|
||||
# get filename from base64-encoded slug
|
||||
filename = b64decode(base64slug).decode("ascii")
|
||||
def results(base64slug):
|
||||
results_html_file = os.path.join(app.config["UPLOAD_PATH"], base64slug)
|
||||
with open(results_html_file, "r") as fh:
|
||||
results_html = fh.read()
|
||||
|
||||
# do we need to use secure_filename again here?
|
||||
input_file = os.path.join(app.config["UPLOAD_PATH"], filename)
|
||||
# write output file with the same name as the input file plus "-cleaned"
|
||||
output_file = os.path.join(
|
||||
app.config["UPLOAD_PATH"], os.path.splitext(filename)[0] + "-cleaned.csv"
|
||||
)
|
||||
|
||||
args = ["-i", input_file, "-o", output_file]
|
||||
|
||||
# run subprocess and capture output as UTF-8 so we get a string instead of
|
||||
# bytes for ansi2html
|
||||
results = subprocess.run(
|
||||
["csv-metadata-quality"] + args,
|
||||
capture_output=True,
|
||||
encoding="UTF-8",
|
||||
)
|
||||
# convert the output to HTML using ansi2html
|
||||
conv = Ansi2HTMLConverter()
|
||||
html = conv.convert(results.stdout)
|
||||
return render_template(
|
||||
"result.html",
|
||||
cli_version=cli_version,
|
||||
filename=filename,
|
||||
stdout=html,
|
||||
base64name=base64slug,
|
||||
)
|
||||
return results_html
|
||||
|
||||
|
||||
@app.route("/result/<base64slug>/download")
|
||||
|
||||
@ -5,13 +5,21 @@
|
||||
{% include 'header.html' %}
|
||||
<main class="flex-shrink-0">
|
||||
<div class="container py-3">
|
||||
<p class="lead">The DSpace CSV Metadata Quality Checker is a pipeline of sanity checks and automated fixes for a number of common issues in metadata files.</p>
|
||||
<p class="lead">The DSpace CSV Metadata Quality Checker is a collection of sanity checks and automated fixes for a number of common issues in metadata files.</p>
|
||||
<form method="POST" action="" enctype="multipart/form-data">
|
||||
<div class="mb-3">
|
||||
<label for="formFile" class="form-label">Select a CSV file to process</label>
|
||||
<input class="form-control" type="file" id="formFile" name="file" accept=".csv">
|
||||
</div>
|
||||
|
||||
<p>Options</p>
|
||||
|
||||
<div class="mb-3 form-check form-switch">
|
||||
<input class="form-check-input" type="checkbox" id="unsafeCheckbox" name="unsafe">
|
||||
<label class="form-check-label" for="unsafeCheckbox" aria-describedby="unsafeHelp">Enable unsafe fixes</label>
|
||||
<div id="unsafeHelp" class="form-text">This will remove newlines and perform <a href="https://withblue.ink/2019/03/11/why-you-need-to-normalize-unicode-strings.html" title='When "Zoë" !== "Zoë". Or why you need to normalize Unicode strings'>normalization of Unicode characters</a>. Read more about these <a href="https://github.com/ilri/csv-metadata-quality#unsafe-fixes">unsafe fixes</a>.</div>
|
||||
</div>
|
||||
|
||||
<button type="submit" class="btn btn-primary">Submit</button>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
@ -4,19 +4,17 @@
|
||||
<body class="d-flex flex-column h-100">
|
||||
{% include 'header.html' %}
|
||||
<main class="flex-shrink-0">
|
||||
<div class="container">
|
||||
<h1 class="mt-3 fs-2">Results</h1>
|
||||
<p>Processing <code>{{ filename }}</code>. Download <a href="/result/{{ base64name }}/download" title="{{ filename | replace('.csv', '-cleaned.csv') }}">cleaned file</a>.</p>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<div class="row justify-content-center">
|
||||
<div class="col-md-10">
|
||||
<div class="container py-3">
|
||||
<p class="lead">The DSpace CSV Metadata Quality Checker is a collection of sanity checks and automated fixes for a number of common issues in metadata files.</p>
|
||||
<h3>Results</h3>
|
||||
<p>Results for <code>{{ filename }}</code>. Download <a href="/result/{{ base64name }}/download" title="{{ filename | replace('.csv', '-cleaned.csv') }}">cleaned file</a>.</p>
|
||||
<h3>Log</h3>
|
||||
<p>The detailed log of the analysis is below. <span style="color: #00aa00">Green</span> indicates a fix was applied, <span style="color: #aa0000">red</span> indicates an error, and <span style="color: #aa5500">orange</span> indicates a warning.</p>
|
||||
<blockquote>
|
||||
{{ stdout | safe }}
|
||||
{{- stdout | safe -}}
|
||||
</blockquote>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
{% include 'footer.html' %}
|
||||
</body>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user