in pp3/au/Runner.php [54:147]
private static function parseLogfile($source) {
$importCounter = 0;
Logger::write(Logger::INFO, 'Starting parsing of the ' . $source . ' logfile ' . self::$grabbedLogFile);
$months = array(
'Jan' => '01',
'Feb' => '02',
'Mar' => '03',
'Apr' => '04',
'May' => '05',
'Jun' => '06',
'Jul' => '07',
'Aug' => '08',
'Sep' => '09',
'Oct' => '10',
'Nov' => '11',
'Dec' => '12'
);
$handle = fopen(self::$grabbedLogFile, 'r');
if ($handle) {
$nl = $nok = 0;
while (($line = fgets($handle)) !== false) {
$hit = false;
// only interested in requests that include the 'unique' identifier as they represent the AU pings
if (!strstr($line, '?unique=') || strstr($line, '/hotfixes/') || strstr($line, '/thirdparty/')) {
$nl++;
continue;
}
switch ($source) {
case 'dlc':
$line=str_replace('<%JSON:httpd_access%> ','', $line);
$jsonLog = json_decode($line, true);
$preg='/^(.+?)\?unique=(unique%3D)?([_A-Z-]+)?(0)?([a-f0-9-]+)(_[a-f0-9-]+)?(.*)?/';
$m = array();
$hit = preg_match($preg, $jsonLog['request'], $match);
if($hit) {
$ip = $jsonLog['clientip'];
$date = date('Y-m-d', strtotime($jsonLog['time']));
$time = date('H:i:s', strtotime($jsonLog['time']));
//$timestamp = $match[1].' '.$match[2];
$path = $jsonLog['uri'];
$product_id = $match[3];
$user_id = $match[5];
if (isset($match[6]) && substr($match[6], 0, 1) == '_') {
$super_id = substr($match[6], 1);
} else {
$super_id = "";
}
$response = $jsonLog['status'];
$bytes = $jsonLog['bytes'];
}
break;
}
// let's parse it and fill these: $ip,$date $time,$path,$product_id,$user_id,$super_id,$response,$bytes
if($hit) {
// check the unique ID format
// prior to NB 5.5.1, the ID was just a timestamp (current time in millis),
// since NB 5.5.1, the ID is a standard UUID: 01234567-89ab-cdef-0123-456789abcdef
if (!preg_match('/^(([0-9]{5,12})|([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}))/', $user_id, $match)) {
Logger::write(Logger::DEBUG, self::$grabbedLogFile . ": suspicious user ID '$user_id' in line: $nl - $line");
} else {
// in 6.5, a truly unique user ID (super ID) has been added,
// check the validity of this ID in UUID format, if available
if (strlen($super_id) > 0 && !preg_match('/^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/', $super_id, $match)) {
Logger::write(Logger::DEBUG, self::$grabbedLogFile . ": suspicious super ID '$super_id' in line: $nl - $line");
// log a warning, but don't stop here, if the super ID is invalid,
// just ignore it
$super_id = "";
}
// return data found, ignore invalid requests (404)
if (strcmp($response, "404")) {
try {
// import it to DB
if (Importer::import(array('ip' => $ip, 'ts' => $date . " " . $time, 'path' => $path, 'distro' => $product_id, 'user_id' => $user_id, 'user2_id' => $super_id, 'response' => $response, 'size' => $bytes), $source) == true) {
$importCounter++;
}
} catch (Exception $e) {
Logger::write(Logger::ERROR, 'Error happened during log entry import: ' . $e->getMessage());
}
$nok++;
}
}
}
$nl++;
}
Logger::write(Logger::INFO, 'Parsed ' . $nl . ' lines of the log, used ' . $nok . ' for importing');
Logger::write(Logger::INFO, 'Really inported into DB were ' . $importCounter . ' AU hits (needed ' . Db::$counter . ' queries)');
} else {
throw new Exception('Unable to open decompressed logfile for parsing ' . self::$grabbedLogFile);
}
return true;
}