automatische Generierung von Keywords/Tags aus einem Text

#use only to set right encoding for browser
 header( 'Content-Type: text/html; charset=UTF-8' );
 
require_once 'text2keywords.class.php';
 
$text="Stoppwörter nennt man im Information Retrieval Wörter, die bei einer Volltextindexierung nicht beachtet werden,
da sie sehr häufig auftreten und gewöhnlich keine Relevanz für die Erfassung des Dokumentinhalts besitzen.
Allgemein übliche Stoppwörter in deutschsprachigen Dokumenten sind bestimmte Artikel ('der', 'die', 'das'),
unbestimmte Artikel ('einer', 'eine', 'ein'), Konjunktionen (z. B. 'und', 'oder', 'doch') und häufig gebrauchte
Präpositionen (z. B. 'an', 'in', 'von'), sowie die Negation 'nicht'. Im Englischen sind unter anderem 'a', 'of',
'the', 'I', 'it', 'you' und 'and' Stoppwörter. Abhängig von den zu erschließenden Dokumenten können Stoppwörter
auch mehrsprachig vorliegen. Obwohl eher als Stoppzeichen zu benennen werden häufig auch der Punkt (.), das Komma
(,) und der Strichpunkt (;) als Stoppwörter bezeichnet.
";
 
$text=mb_convert_encoding( $text, "utf-8");
$tags=new text2keywords(array("stopwords/de_stopwords.txt","stopwords/vbnet_stopwords.txt|,"));
 
#$tags->myUnwanted=array('some','words','you','dont','want');
#$tags->FilterWords=true; #filter out words < $minFilter
#$tags->myUnwanted=3 #set min word len default 3; only used when FilterWords=true
 
$myTags=$tags->GetTags($text);
print_r($myTags);

Array
(
    [stoppwörter] => 5
    [häufig] => 3
    [artikel] => 2
    [dokumenten] => 2
    [englischen] => 1
    [...]
}

/**
 * Class that will return the tags/keywords of the given text
 * 
 * Add Stopfile words as array: "file|[splitter]" (default:splitter=newline)
 * array("de_stopwords.txt") ' each has own line, separated by \r\n
 * array("de_stopwords.txt|,") 'words separated by ,
 * 
 * @param array $stopFiles
 */
class text2keywords {
	/**
	 * strim some chars like ,:;
	 *
	 * @var boolean
	 */
	public $strip_chars =true;
 
	/**
	 * extra words that should not count
	 *
	 * @var array
	 */
	public $myUnwanted =array();
 
	/**
	 * minimal word long, short will removed
	 *
	 * @var integer
	 */
	public $minString=3;
 
	/**
	 * Filter words on len an trim it
	 *
	 * @var boolean
	 */
	public $FilterWords=false;
 
	private $_stopWords =array();
 
	/**
	 * Return tags filtered by stop/unwated words and ordered by usage
	 *
	 * @param string $text
	 * @return array
	 */
	function GetTags($text) {
			$text=strtolower($text);
			if ($this->strip_chars==true) $text=$this->f_strip_chars($text);
			$words=$this->SplitWords($text,' +');
			if (count($this->_stopWords)>0) $words = array_diff($words, $this->_stopWords);
			if (count($this->myUnwanted)>0)	$words = array_diff($words, $this->myUnwanted);
 
			if ($this->FilterWords==true) $words=$this->f_FilterWords($words); 
 
			$keywordCounts = array_count_values( $words );
			arsort( $keywordCounts, SORT_NUMERIC );
			return $keywordCounts;
	}
 
	private function f_FilterWords($words) {
		$words1=array();
		foreach($words as $word) {
			$word=trim($word);
			if (strlen($word)>=$this->minString) {
				$words1[]=trim($word);
			}
		}
		return $words1;		
	}
 
	/**
	 * return all words in raw foramt
	 *
	 * @param string $text
	 * @return array
	 */
	function GetAllWords($text) {
		if ($this->strip_chars==true) $text=$this->f_strip_chars($text);
		return $this->SplitWords($text,' +');
	}
 
	/**
	 * Add Stopfile words array: "file|[splitter]" (default:splitter=newline)
	 * array("de_stopwords.txt") ' each has own line, separated by \r\n
	 * array("de_stopwords.txt|,") 'words separated by ,
	 * 
	 * @param array $stopFiles
	 */
	function __construct($stopFiles=Array(),$TrimClean=false) {
		$words1=array();
		if (count($stopFiles)>0) {
			foreach($stopFiles as $key=>$file) {
				$split=$this->GetSplitter($file);
				if ($split=="") {
					$wordsfile=$this->ReadLineFromFile($file);
					$words=$this->SplitWords($wordsfile);
				} else {
					$wordsfile=$this->ReadLineFromFile(str_replace('|'.$split,"",$file));
					$words=$this->SplitWords($wordsfile,$split);
				}
 
				$this->_stopWords =array_merge($this->_stopWords,$words);
			}
		}
	}
 
	private function f_strip_chars($text) {
		$text= preg_replace("/\r\n|\r|\n/"," ",$text);
		return preg_replace("/\.|,|:|;|-|'|\(|\)|\"/"," ",$text);
	}	
 
	private function GetSplitter($txt) {
		if (strlen(strstr($txt,'|'))>0) {
			return str_replace("|","",strstr($txt,'|'));
		}
	return "";
	}
 
	private function SplitWords($text,$splitter="\r\n") {
		$text=strtolower($text);
		return mb_split($splitter, $text);
	}
 
	private function ReadLineFromFile($datei) {
		if (!file_exists($datei)) return "";
		$back="";
		$fhandler=fopen($datei,"r");
		while(!feof($fhandler))	{
			$tHelp=fgets($fhandler);
			if ($this->StartsWithComment($tHelp)==false) $back .= $tHelp;
		}
		fclose($fhandler);
		return $back;
	}
 
	/**
	 * StartsWithComamnd
	 * Tests if a line starts with space,',*,#,/,; or \.
	 *
	 * @param     string
	 * @param     string
	 * @return    bool
	 */
	private function StartsWithComment($str){
		return preg_match('/^[ |\'|*|#|\/|;|\\\]/', $str);
	}
}