plagiator/Shingles.php at master · StackThrower/plagiator · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
<?php


namespace TextComparator;


abstract class Shingles
{
    const TABS_AND_NEWLINES_REGEX = '!\s+!';

    /**
     * @var array
     */
    protected $stopSymbols = ['.', ';', '-', ','];


    /**
     * @var integer
     */
    protected $shinglesCount;

    /**
     * @param int $shinglesCount
     */
    public function __construct($shinglesCount = 3)
    {
        if (!is_int($shinglesCount)) {
            throw new \InvalidArgumentException('$shinglesCount must be int type');
        }

        $this->shinglesCount = $shinglesCount;
    }

    /**
     * @param $content
     * @return array
     */
    public function canonize($content)
    {
        $content = strtolower($content);
        $content = str_replace($this->stopSymbols, '', $content);
        $content = preg_replace(self::TABS_AND_NEWLINES_REGEX, ' ', $content);

        return explode(' ', $content);
    }

    /**
     * @param array $words
     * @return array
     */
    public function splitShingles(array $words)
    {
        $shingles = array();

        $countWords = count($words);

        for ($i = 0; $i < $countWords - $this->shinglesCount; $i++) {

            $shingles[] = $this->makeHash(implode(' ', array_slice($words, $i, $this->shinglesCount)));
        }

        return $shingles;
    }

    /**
     * @param string $shingle
     * @return mixed
     */
    abstract public function makeHash($shingle);

    /**
     * @param $content1
     * @param $content2
     * @return float
     */
    public function compare($content1, $content2)
    {
        $shingle1 = $this->getShingles($content1);
        $shingle2 = $this->getShingles($content2);

        $diff = array_diff($shingle1, $shingle2);
        $count_shingle = count($shingle1);

        $uniqueness =  count($diff) / ($count_shingle / 100); // уникальность %

        return $uniqueness;
    }

    /**
     * @param $text
     * @return string
     */
    public function getShinglesAsString($text)
    {
        $shingles = $this->splitShingles($this->canonize($text));

        return implode(' ', $shingles);
    }

    /**
     * @param $text
     * @return string
     */
    public function getShingles($text)
    {
        $shingles = $this->splitShingles($this->canonize($text));

        return $shingles;
    }

    /**
     * @return array
     */
    public function getStopSymbols()
    {
        return $this->stopSymbols;
    }

    /**
     * @param array $stopSymbols
     */
    public function addStopSymbols(array $stopSymbols)
    {
        $this->stopSymbols = $stopSymbols;
    }
}