采用Nagao算法统计各个子字符串的频次，然后基于这些频次统计每个字符串的词频、左右邻个数、左右熵、交互信息(内部凝聚度)。
名词解释：
Nagao算法：一种快速的统计文本里所有子字符串频次的算法。详细算法可见http://www.doc88.com/p-664123446503.html
词频：该字符串在文档中出现的次数。出现次数越多越重要。
左右邻个数：文档中该字符串的左边和右边出现的不同的字的个数。左右邻越多，说明字符串成词概率越高。
左右熵：文档中该字符串的左边和右边出现的不同的字的数量分布的熵。类似上面的指标，有一定区别。
交互信息：每次将某字符串分成两部分，左半部分字符串和右半部分字符串，计算其同时出现的概率除于其各自独立出现的概率，最后取所有的划分里面概率最小值。这个值越大，说明字符串内部凝聚度越高，越可能成词。
算法具体流程：
1. 将输入文件逐行读入，按照非汉字([^一-龥]+)以及停词“的很了么呢是嘛个都也比还这于不与才上用就好在和对挺去后没说”，
分成一个个字符串，代码如下：
停用词可以修改。
2. 获取所有切分后的字符串的左子串和右子串，分别加入左、右PTable
3. 对PTable排序，并计算LTable。LTable记录的是，排序后的PTable中，下一个子串同上一个子串具有相同字符的数量
4. 遍历PTable和LTable，即可得到所有子字符串的词频、左右邻
5. 根据所有子字符串的词频、左右邻结果，输出字符串的词频、左右邻个数、左右熵、交互信息
1. NagaoAlgorithm.java
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
public class NagaoAlgorithm {
private int N;
private List leftPTable;
private int[] leftLTable;
private List rightPTable;
private int[] rightLTable;
private double wordNumber;
private Map wordTFNeighbor;
private final static String stopwords = "的很了么呢是嘛个都也比还这于不与才上用就好在和对挺去后没说";
private NagaoAlgorithm(){
//default N = 5
N = 5;
leftPTable = new ArrayList();
rightPTable = new ArrayList();
wordTFNeighbor = new HashMap();
//reverse phrase
private String reverse(String phrase) {
StringBuilder reversePhrase = new StringBuilder();
for (int i = phrase.length() - 1; i >= 0; i--)
reversePhrase.append(phrase.charAt(i));
return reversePhrase.toString();
//co-prefix length of s1 and s2
private int coPrefixLength(String s1, String s2){
int coPrefixLength = 0;
for(int i = 0; i < Math.min(s1.length(), s2.length()); i++){
if(s1.charAt(i) == s2.charAt(i)) coPrefixLength++;
return coPrefixLength;
//add substring of line to pTable
private void addToPTable(String line){
//split line according to consecutive none Chinese character
for(String phrase : phrases){
for(int i = 0; i < phrase.length(); i++)
rightPTable.add(phrase.substring(i));
String reversePhrase = reverse(phrase);
for(int i = 0; i < reversePhrase.length(); i++)
leftPTable.add(reversePhrase.substring(i));
wordNumber += phrase.length();
//count lTable
private void countLTable(){
Collections.sort(rightPTable);
rightLTable = new int[rightPTable.size()];
for(int i = 1; i < rightPTable.size(); i++)
rightLTable[i] = coPrefixLength(rightPTable.get(i-1), rightPTable.get(i));
Collections.sort(leftPTable);
leftLTable = new int[leftPTable.size()];
for(int i = 1; i < leftPTable.size(); i++)
leftLTable[i] = coPrefixLength(leftPTable.get(i-1), leftPTable.get(i));
System.out.println("Info: [Nagao Algorithm Step 2]: having sorted PTable and counted left and right LTable");
//according to pTable and lTable, count statistical result: TF, neighbor distribution
private void countTFNeighbor(){
//get TF and right neighbor
for(int pIndex = 0; pIndex < rightPTable.size(); pIndex++){
String phrase = rightPTable.get(pIndex);
for(int length = 1 + rightLTable[pIndex]; length <= N && length <= phrase.length(); length++){
String word = phrase.substring(0, length);
TFNeighbor tfNeighbor = new TFNeighbor();
tfNeighbor.addToRightNeighbor(phrase.charAt(length));
for(int lIndex = pIndex+1; lIndex < rightLTable.length; lIndex++){
if(rightLTable[lIndex] >= length){
String coPhrase = rightPTable.get(lIndex);
tfNeighbor.addToRightNeighbor(coPhrase.charAt(length));
wordTFNeighbor.put(word, tfNeighbor);
//get left neighbor
for(int pIndex = 0; pIndex < leftPTable.size(); pIndex++){
String phrase = leftPTable.get(pIndex);
for(int length = 1 + leftLTable[pIndex]; length <= N && length <= phrase.length(); length++){
String word = reverse(phrase.substring(0, length));
TFNeighbor tfNeighbor = wordTFNeighbor.get(word);
tfNeighbor.addToLeftNeighbor(phrase.charAt(length));
for(int lIndex = pIndex + 1; lIndex < leftLTable.length; lIndex++){
if(leftLTable[lIndex] >= length){
String coPhrase = leftPTable.get(lIndex);
tfNeighbor.addToLeftNeighbor(coPhrase.charAt(length));
System.out.println("Info: [Nagao Algorithm Step 3]: having counted TF and Neighbor");
//according to wordTFNeighbor, count MI of word
private double countMI(String word){
if(word.length() <= 1) return 0;
double coProbability = wordTFNeighbor.get(word).getTF()/wordNumber;
List mi = new ArrayList(word.length());
for(int pos = 1; pos < word.length(); pos++){
String leftPart = word.substring(0, pos);
String rightPart = word.substring(pos);
double leftProbability = wordTFNeighbor.get(leftPart).getTF()/wordNumber;
double rightProbability = wordTFNeighbor.get(rightPart).getTF()/wordNumber;
mi.add(coProbability/(leftProbability*rightProbability));
return Collections.min(mi);
//save TF, (left and right) neighbor number, neighbor entropy, mutual information
private void saveTFNeighborInfoMI(String out, String stopList, String[] threshold){
//read stop words file
Set stopWords = new HashSet();
BufferedReader br = new BufferedReader(new FileReader(stopList));
if(line.length() > 1)
stopWords.add(line);
//output words TF, neighbor info, MI
BufferedWriter bw = new BufferedWriter(new FileWriter(out));
for(Map.Entry entry : wordTFNeighbor.entrySet()){
if( entry.getKey().length() <= 1 || stopWords.contains(entry.getKey()) ) continue;
TFNeighbor tfNeighbor = entry.getValue();
int tf, leftNeighborNumber, rightNeighborNumber;
double mi;
tf = tfNeighbor.getTF();
leftNeighborNumber = tfNeighbor.getLeftNeighborNumber();
rightNeighborNumber = tfNeighbor.getRightNeighborNumber();
mi = countMI(entry.getKey());
if(tf > Integer.parseInt(threshold[0]) && leftNeighborNumber > Integer.parseInt(threshold[1]) &&
rightNeighborNumber > Integer.parseInt(threshold[2]) && mi > Integer.parseInt(threshold[3]) ){
StringBuilder sb = new StringBuilder();
sb.append(entry.getKey());
sb.append(",").append(tf);
sb.append(",").append(leftNeighborNumber);
sb.append(",").append(rightNeighborNumber);
sb.append(",").append(tfNeighbor.getLeftNeighborEntropy());
sb.append(",").append(tfNeighbor.getRightNeighborEntropy());
sb.append(",").append(mi).append(" ");
bw.write(sb.toString());
bw.close();
throw new RuntimeException(e);
System.out.println("Info: [Nagao Algorithm Step 4]: having saved to file");
//apply nagao algorithm to input file
public static void applyNagao(String[] inputs, String out, String stopList){
nagao.saveTFNeighborInfoMI(out, stopList, "20,3,3,5".split(","));
public static void applyNagao(String[] inputs, String out, String stopList, int n, String filter){
nagao.setN(n);
String[] threshold = filter.split(",");
if(threshold.length != 4){
System.out.println("ERROR: filter must have 4 numbers, seperated with ',' ");
return;
nagao.saveTFNeighborInfoMI(out, stopList, threshold);
private void setN(int n){
N = n;
String[] ins = {"E://test//ganfen.txt"};
applyNagao(ins, "E://test//out.txt", "E://test//stoplist.txt");
2. TFNeighbor.java
public class TFNeighbor {
private int tf;
private Map leftNeighbor;
private Map rightNeighbor;
TFNeighbor(){
leftNeighbor = new HashMap();
rightNeighbor = new HashMap();
//add word to leftNeighbor
public void addToLeftNeighbor(char word){
//leftNeighbor.put(word, 1 + leftNeighbor.getOrDefault(word, 0));
Integer number = leftNeighbor.get(word);
leftNeighbor.put(word, number == null? 1: 1+number);
//add word to rightNeighbor
public void addToRightNeighbor(char word){
//rightNeighbor.put(word, 1 + rightNeighbor.getOrDefault(word, 0));
Integer number = rightNeighbor.get(word);
rightNeighbor.put(word, number == null? 1: 1+number);
//increment tf
public void incrementTF(){
tf++;
public int getLeftNeighborNumber(){
return leftNeighbor.size();
public int getRightNeighborNumber(){
return rightNeighbor.size();
public double getLeftNeighborEntropy(){
for(int number : leftNeighbor.values()){
public double getRightNeighborEntropy(){
for(int number : rightNeighbor.values()){
public int getTF(){
return tf;
3. Main.java
public class Main {
//if 3 arguments, first argument is input files splitting with ','
//second argument is output file
//output 7 columns split with ',' , like below:
//word, term frequency, left neighbor number, right neighbor number, left neighbor entropy, right neighbor entropy, mutual information
//third argument is stop words list
if(args.length == 3)
NagaoAlgorithm.applyNagao(args[0].split(","), args[1], args[2]);
//if 4 arguments, forth argument is the NGram parameter N
//5th argument is threshold of output words, default is "20,3,3,5"
//output TF > 20 && (left | right) neighbor number > 3 && MI > 5
else if(args.length == 5)
NagaoAlgorithm.applyNagao(args[0].split(","), args[1], args[2], Integer.parseInt(args[3]), args[4]);
本文地址：http://www.dbeile.cn/quote/102.html 多贝乐 http://www.dbeile.cn/ , 查看更多