Crispi/dogstats
Archived
2
0
Fork 0
Code Issues 1 Pull Requests Projects Wiki Activity

smart test
All checks were successful
Build and push / Pulling repo on server (push) Successful in 3s
Details

Browse Source
This commit is contained in:
Chris 2023-11-26 10:30:00 +01:00
parent e944c552ba
commit 8bc91a14f5
7 changed files with 185 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
crawler/.gitattributes vendored
View File

@ -1,2 +0,0 @@
*.pdf filter=lfs diff=lfs merge=lfs -text
*.csv filter=lfs diff=lfs merge=lfs -text

39
crawler/base.sql Normal file
View File

@ -0,0 +1,39 @@
BEGIN TRANSACTION;
CREATE TABLE IF NOT EXISTS "events" (
"ID" INTEGER UNIQUE,
"name" TEXT,
"date" TEXT,
"organizer" TEXT,
PRIMARY KEY("ID")
);
CREATE TABLE IF NOT EXISTS "runs" (
"id" INTEGER UNIQUE,
"event" INTEGER,
"name" TEXT,
"lk" TEXT,
"gk" TEXT,
FOREIGN KEY("event") REFERENCES "events"("ID"),
PRIMARY KEY("id")
);
CREATE TABLE IF NOT EXISTS "results" (
"id" INTEGER,
"event" INTEGER,
"run" INTEGER,
"rang" TEXT,
"stnr" INTEGER,
"teilnehmer" TEXT,
"hund" TEXT,
"verein" TEXT,
"f" INTEGER,
"vw" INTEGER,
"zf" INTEGER,
"zeit" REAL,
"gf" REAL,
"msek" REAL,
"bew" TEXT,
"punkte" TEXT,
PRIMARY KEY("id" AUTOINCREMENT),
FOREIGN KEY("run") REFERENCES "runs"("id"),
FOREIGN KEY("event") REFERENCES "events"("ID")
);
COMMIT;

64
crawler/crawler.php
View File

@ -3,6 +3,11 @@
//$ergebnisse = file_get_contents('https://www.dognow.at/ergebnisse/'); //$ergebnisse = file_get_contents('https://www.dognow.at/ergebnisse/');
//file_put_contents('tmp/ergebnisse.html', $ergebnisse); //file_put_contents('tmp/ergebnisse.html', $ergebnisse);
$GLOBALS['db'] = new SQLite3('data.db');
if(!$GLOBALS['db']) exit("Error loading database");
var_dump($GLOBALS['db']);
$html = file_get_contents('tmp/ergebnisse.html'); $html = file_get_contents('tmp/ergebnisse.html');
$dom = new DOMDocument; $dom = new DOMDocument;
@ -52,13 +57,18 @@ foreach($pages as $key=> $page){
// Loop through the selected nodes // Loop through the selected nodes
foreach ($nodes as $node) { foreach ($nodes as $node) {
// Do something with each node, for example, echo its content // CUPs
//echo $node->nodeValue . "\n";
$div = $node->getElementsByTagName('div')[0]; $div = $node->getElementsByTagName('div')[0];
$id = $div->getAttribute('data-event'); $id = $div->getAttribute('data-event');
$name = trim($div->getElementsByTagName('div')[1]->nodeValue); $name = trim($div->getElementsByTagName('div')[1]->nodeValue);
$organizer = trim($div->getElementsByTagName('div')[2]->nodeValue); $organizer = trim($div->getElementsByTagName('div')[2]->nodeValue);
$date = trim($div->getElementsByTagName('div')[3]->nodeValue); $date = trim($div->getElementsByTagName('div')[3]->nodeValue);
$db_date = date(DATE_RFC3339, strtotime($date));
//if not exists, add to db
$res = $GLOBALS['db']->query("SELECT * FROM events WHERE id = '$id'");
if($res->fetchArray() == false)
$GLOBALS['db']->exec("INSERT INTO events (id, name, organizer, date) VALUES ('$id', '$name', '$organizer', '$db_date')");
crawlRuns($id); crawlRuns($id);
@ -124,6 +134,12 @@ Derzeit sind keine Ergebnisse")!==false)
$gk = trim($tds[2]->nodeValue); $gk = trim($tds[2]->nodeValue);
$pdf = $tds[3]->getElementsByTagName('a')[0]->getAttribute('href'); $pdf = $tds[3]->getElementsByTagName('a')[0]->getAttribute('href');
//add run to db if not exists
$res = $GLOBALS['db']->query("SELECT * FROM runs WHERE id = '$rid'");
if($res->fetchArray() == false)
$GLOBALS['db']->exec("INSERT INTO runs (id, name, event, lk, gk) VALUES ('$rid', '$runname', '$eventid', '$lk', '$gk')");
getResults($rid,$eventid); getResults($rid,$eventid);
} }
@ -146,6 +162,50 @@ function getResults($run,$event)
if(!file_exists('tmp/results/' . $event . '-' . $run . '.pdf')) if(!file_exists('tmp/results/' . $event . '-' . $run . '.pdf'))
file_put_contents('tmp/results/' . $event . '-' . $run . '.pdf',file_get_contents($url)); file_put_contents('tmp/results/' . $event . '-' . $run . '.pdf',file_get_contents($url));
convertPDFtoCSV('tmp/results/' . $event . '-' . $run . '.pdf','tmp/csv/' . $event . '-' . $run . '.pdf.csv'); convertPDFtoCSV('tmp/results/' . $event . '-' . $run . '.pdf','tmp/csv/' . $event . '-' . $run . '.pdf.csv');
analyzeResultCSV('tmp/csv/' . $event . '-' . $run . '.pdf.csv',$run,$event);
}
function analyzeResultCSV($csv,$run,$event)
{
$csv = array_map('str_getcsv', file($csv));
//prepare header for database
foreach($csv[0] as $key=>$value){
$csv[0][$key] = preg_replace('/[^A-Za-z0-9]/', '', strtolower($value));
}
array_walk($csv, function(&$a) use ($csv) {
$a = array_combine($csv[0], $a);
});
array_shift($csv); # remove column header
foreach($csv as $row)
{
$stnr = $row['stnr'];
$teilnehmer = $row['teilnehmer'];
$hund = $row['hund'];
$verein = $row['verein'];
$f = $row['f'];
$vw = $row['vw'];
$zf = $row['zf'];
$zeit = $row['zeit'];
$gf = $row['gf'];
$msek = $row['msek'];
$bew = $row['bew'];
//add result to db if not exists
try
{
$res = $GLOBALS['db']->query("SELECT * FROM results WHERE stnr = '$stnr' AND run = '$run' AND event = '$event'");
if($res->fetchArray() == false)
$GLOBALS['db']->exec("INSERT INTO results (stnr, run, event, teilnehmer, hund, verein, f, vw, zf, zeit, gf, msek, bew) VALUES ('$stnr', '$run', '$event', '$teilnehmer', '$hund', '$verein', '$f', '$vw', '$zf', '$zeit', '$gf', '$msek', '$bew')");
}
catch(Exception $ex) {
//die( $ex->getMessage() );
exit($GLOBALS['db']->lastErrorMsg());
}
}
} }
function convertPDFtoCSV($pdf,$targetname) function convertPDFtoCSV($pdf,$targetname)

BIN
crawler/data.db Normal file
View File

Binary file not shown.

33
web/pages/smart/controller.php Normal file
View File

@ -0,0 +1,33 @@
<?php
class Smart extends Page {
function setMenu()
{
$this->menu_text = 'Smart';
$this->menu_image = 'fas fa-robot';
$this->menu_priority = 1;
}
function index()
{
$this->set('template', "smart.html.php");
}
function search()
{
$db = new SQLite3(ROOT.DS.'../crawler/data.db');
$q = $_REQUEST['q'];
$res = $db->query("SELECT * FROM results WHERE teilnehmer LIKE '$q'");
$results = [];
while($row = $res->fetchArray())
{
$results[] = $row;
}
$this->set('results', $results);
$this->set('template', 'search.html.php');
}
}

40
web/pages/smart/search.html.php Normal file
View File

@ -0,0 +1,40 @@
<table class="table">
<tr>
<th>event</th>
<th>run</th>
<th>rang</th>
<th>stnr</th>
<th>teilnehmer</th>
<th>hund</th>
<th>verein</th>
<th>f</th>
<th>vw</th>
<th>zf</th>
<th>zeit</th>
<th>gf</th>
<th>msek</th>
<th>bew</th>
<th>punkte</th>
</tr>
<?php foreach($results as $res): ?>
<tr>
<td><?= $res['event'] ?></td>
<td><?= $res['run'] ?></td>
<td><?= $res['rang'] ?></td>
<td><?= $res['stnr'] ?></td>
<td><?= $res['teilnehmer'] ?></td>
<td><?= $res['hund'] ?></td>
<td><?= $res['verein'] ?></td>
<td><?= $res['f'] ?></td>
<td><?= $res['vw'] ?></td>
<td><?= $res['zf'] ?></td>
<td><?= $res['zeit'] ?></td>
<td><?= $res['gf'] ?></td>
<td><?= $res['msek'] ?></td>
<td><?= $res['bew'] ?></td>
<td><?= $res['punkte'] ?></td>
</tr>
<?php endforeach; ?>
</table>

11
web/pages/smart/smart.html.php Normal file
View File

@ -0,0 +1,11 @@
<h2>Dein Name wie der auf Teilnahmelisten zu finden ist</h2>
<input type="text" name="q"
hx-get="/smart/search"
hx-trigger="keyup changed delay:500ms"
hx-target="#search-results"
placeholder="Suchen..."
hx-indicator="#indicator"
>
<div id="search-results"></div>
<i id="indicator" class="fad fa-spinner-third fa-spin htmx-indicator"></i>