<?php

/*
 *  score.inc.phps (Markov and Sinkov score functions)
 *  Copyright (C) 2018 Daniel Marschall, ViaThinkSoft
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software Foundation,
 *  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 */

$cache_sinkov = array();
$cache_markov = array();

/**
    sinkov_score(): Calculates the sinkov score of a given text
    @param $text   The input text
    @param $lang   The language code
    @param $n      2 for bigrams, 3 for trigrams, 4 for quadgrams
    @param $strict If true, the score will be zero if a single n-gram has the probability of 0.
**/
function sinkov_score($text, $lang='en', $n=2, $strict=false) {
	global $cache_sinkov;

	$cache_key  = $lang.$n;
	$cache_file = __DIR__ . "/stats/sinkov_{$lang}_{$n}.csv";
	if (!isset($cache_sinkov[$cache_key])) {
		if (!file_exists($cache_file)) {
			throw new Exception("Sinkov statistics file $cache_file does not exist!");
		}

		$handle = fopen($cache_file, "r");
		if (!$handle) {
			throw new Exception("Cannot read Sinkov statistics file $cache_file!");
		}

		$cache_sinkov[$cache_key] = array();
		while (($line = fgets($handle)) !== false) {
			$line = trim($line);
			if ($line == '') continue;
			list($gram, $val) = explode(',', $line);
			$cache_sinkov[$cache_key][$gram] = $val;
		}
		fclose($handle);
	}

	$text = str_replace(array("\r", "\n", "\t"), ' ', trim($text));
	$words = explode(' ', $text);
	if (count($words) > 1) {
		$count = 0;
		$sum = 0;
		foreach ($words as $word) {
			if ($word == '') continue;
			$count++;
			$sum += sinkov_score($word, $lang, $n);
		}
		return $count == 0 ? 0 : $sum/$count;
	} else {
		$text = strtolower($text);
		$text = preg_replace('@[^a-z]+@', '', $text);

		$score = 0;
		$count = 0;
		for ($i=0; $i<strlen($text)-($n-1); $i++) {
			$gram = substr($text, $i, $n);
			if (isset($cache_sinkov[$cache_key][$gram])) {
				$loc_score = $cache_sinkov[$cache_key][$gram];
				if ($strict && ($loc_score == 0)) return 0;
				$score += $loc_score;
				$count++;
			}
		}

		// TODO: if we are trying to find out the n=5 score of a word with size 3, nothing is found. Should we sum up all possible grams which have the prefix of our wanted word?
		return $count == 0 ? 0 : $score/$count;
	}
}

/**
    markov_score(): Calculates the markov score of a given text
    @param $text     The input text
    @param $lang     The language code
    @param $n        2 for bigrams, 3 for trigrams, 4 for quadgrams
    @param $multiply If true, the markov scores will be multiplied (e.g. a*b*c);
                     This allows you to determine how likely it is that a text is a valid word.
                     If false (default), the markov scores will be added (e.g. a+b+c / 3);
                     This is is recommended if you have a whole text without whitespaces.
**/
function markov_score($text, $lang='en', $n=2, $multiply=false) {
	global $cache_markov;

	$cache_key  = $lang.$n;
	$cache_file = __DIR__ . "/stats/markov_{$lang}_{$n}.csv";
	if (!isset($cache_markov[$cache_key])) {
		if (!file_exists($cache_file)) {
			throw new Exception("Markov statistics file $cache_file does not exist!");
		}

		$handle = fopen($cache_file, "r");
		if (!$handle) {
			throw new Exception("Cannot read Markov statistics file $cache_file!");
		}

		$cache_markov[$cache_key] = array();
		while (($line = fgets($handle)) !== false) {
			$line = trim($line);
			if ($line == '') continue;
			list($gram, $val) = explode(',', $line);
			$cache_markov[$cache_key][$gram] = $val;
		}
		fclose($handle);
	}

	$text = str_replace(array("\r", "\n", "\t"), ' ', trim($text));
	$words = explode(' ', $text);
	if (count($words) > 1) {
		$count = 0;
		$sum = 0;
		foreach ($words as $word) {
			if ($word == '') continue;
			$count++;
			$sum += markov_score($word, $lang, $n, $multiply);
		}
		return $count == 0 ? 0 : $sum/$count;
	} else {
		$text = strtolower($text);
		$text = preg_replace('@[^a-z]+@', '', $text);

		$score = $multiply ? 1 : 0;
		$count = 0;
		for ($i=0; $i<strlen($text)-($n-1); $i++) {
			$gram = substr($text, $i, $n);
			if (isset($cache_markov[$cache_key][$gram])) {
				$score = $multiply ? $score*$cache_markov[$cache_key][$gram] : $score+$cache_markov[$cache_key][$gram];
				$count++;
			}
		}

		// TODO: if we are trying to find out the n=5 score of a word with size 3, nothing is found. Should we sum up all possible grams which have the prefix of our wanted word?
		return $multiply ? $score : ($count == 0 ? : $score/$count);
	}
}

/*
echo "sinkov2 good text       : ".sinkov_score('hellothisisashorttextwithvalidenglish')."\n";
echo "sinkov2 bad text        : ".sinkov_score('isdmnvzkvzshfoiezvjhpeshvtzbhjflspfin')."\n";

echo "normal markov2 good text: ".markov_score('hellothisisashorttextwithvalidenglish')."\n";
echo "normal markov2 bad text : ".markov_score('isdmnvzkvzshfoiezvjhpeshvtzbhjflspfin')."\n";

echo "strict markov2 good text: ".markov_score('hellothisisashorttextwithvalidenglish', 'en', 2, true)."\n";
echo "strict markov2 bad text : ".markov_score('isdmnvzkvzshfoiezvjhpeshvtzbhjflspfin', 'en', 2, true)."\n";
*/