'''
setup.py file for NLPIR
'''
from distutils.core import setup, Extension

NLPIR_module = Extension('_NLPIR',sources=['NLPIR_wrap.cxx'], libraries = ['NLPIR'])

setup(name = 'NLPIR',
    version = '1.1',
    author = 'Killua',
    description = 'Python for NLPIR',
    ext_modules = [NLPIR_module],
    py_modules = ['NLPIR'],
    )

python setup.py build_ext --inplace

#!/usr/bin/env python
#coding:utf-8
'''
Created on Fri 19, 2013

@author: killua
@e-mail: [email protected]
@Decription: Python for NLPIR
'''
import NLPIR
import os

def nlpir_init(init_dir = '.', code_type = 'GBK'):
    '''
    Init the analyzer and prepare necessary data for NLPIR according the configure file.
    '''
    if code_type == 'GBK':
        is_succeed = NLPIR.NLPIR_Init(init_dir, NLPIR.GBK_CODE)
    elif code_type == 'UTF-8' or code_type == 'UTF8':
        is_succeed = NLPIR.NLPIR_Init(init_dir, NLPIR.UTF8_CODE)
    elif code_type == 'BIG5':
        is_succeed = NLPIR.NLPIR_Init(init_dir, NLPIR.BIG5_CODE)
    elif code_type == 'GBK_FANTI':
        is_succeed = NLPIR.NLPIR_Init(init_dir, NLPIR.GBK_FANTI_CODE)
    if is_succeed:
        print 'NLPIR Successful.'
    else:
        print 'NLPIR Failed.'

def nlpir_exit():
    '''
    Exit the program and free all resources and destroy all working buffer used in NLPIR.
    '''
    return NLPIR.NLPIR_Exit()

def nlpir_import_user_dict(user_dict):
    '''
    Import user-defined dictionary from a text file.
    '''
    return NLPIR.NLPIR_ImportUserDict(user_dict)

def nlpir_paragraph_process(text, is_pos_tagged = False):
    '''
    Process a paragraph
    '''
    return NLPIR.NLPIR_ParagraphProcess(text, is_pos_tagged)

def nlpir_file_process(source_file, target_file, is_pos_tagged = False):
    '''
    Process a text file
    '''
    return NLPIR.NLPIR_FileProcess(source_file, target_file, is_pos_tagged)

def nlpir_add_user_word(word):
    '''
    Add a word to the user dictionary.
    '''
    return NLPIR.NLPIR_AddUserWord(word)

def nlpir_save_user_dict():
    '''
    Save the user dictionary to disk.
    '''
    return NLPIR.NLPIR_SaveTheUsrDic()

def nlpir_delete_user_word(word):
    '''
    Delete a word from the  user dictionary.
    '''
    return NLPIR.NLPIR_DelUsrWord(word)

#!/usr/bin/env python
#encoding: utf8

from PyNLPIR import *

if __name__ == '__main__':

    nlpir_init('.', 'UTF-8')
    print nlpir_paragraph_process(r'@ICTCLAS张华平博士 应各位ICTCLAS用户的要求，张华平博士提前发布ICTCLAS2013 版本，为了与以前工作进行大的区隔，并推广NLPIR自然语言处理与信息检索共享平台，从本版本开始，系统名称调整为NLPIR汉语分词系统。')
    print
    print nlpir_paragraph_process(r'“屌丝”这个嘲讽意味的代词迅速爆红，迎合了大众的心理和趣味。因为你会发现从表面符合屌丝定义的人，到和屌丝属性八竿子打不着的人，都在争相认领这一名号。当人人都在忙着确认自己的屌丝身份，并乐此不疲时，屌丝一词一定与时代的什么特征实现了合拍。“屌丝”不是阿Q，他们公然比惨并乐在其中有评论认为，“屌丝”是新时代的阿Q，两者并不完全相同。首先，阿Q是文学巨匠鲁迅一己之力创造的，而“屌丝”则是网络群体狂欢的结果，它是真正由网民集体创作的形象；另外，阿Q最重要的特征是“精神胜利法”，梦想的是“银盔银甲”，意淫的是“我手持钢鞭将你打”。', True)
    nlpir_exit()

DICTIONARY = {}

def read_words():
    '''
    Read bag of words from file
    '''
    f = open('wiktionary', 'r')
    for line in f.readlines():
        DICTIONARY.update({line.split(':')[0] : float(line.split(':')[1])})

def generate_edit_distance1_words(word):
    '''
    Generate a set of all words that are one edit distance from word
    '''
    #delete a character for word
    deletes = [word[1:]]
    deletes += [str(word[:i] + word[i+1:]) for i in range(1, len(word))]
    #change position between two character for word
    transposes = [str(word[1] + word[0] + word[2:])]
    transposes += [str(word[:i-1] + word[i] + word[i-1] + word[i+1:]) for i in range(2, len(word))]
    #replaces one character
    replaces = [str(c + word[1:]) for c in string.lowercase]
    replaces += [str(word[:i] + c + word[i+1:]) for i in range(1, len(word)) for c in string.lowercase]
    #insert one character
    inserts = [str(c + word) for c in string.lowercase]
    inserts += [str(word[:i] + c + word[i:]) for i in range(1, len(word)) for c in string.lowercase]
    inserts += [str(word + c) for c in string.lowercase]

    return set(deletes + transposes + replaces +inserts)

def words_filter(words):
    '''
    Word filter
    '''
    return set(word for word in words if word in DICTIONARY.keys())

def candidates(word):
    '''
    Get all candidates for word
    '''
    if word in DICTIONARY.keys():
        return set([word])
    else:
        return words_filter([word]) | words_filter(generate_edit_distance1_words(word))

def correct(word):
    '''
    correct the word.
    '''
    candidate_words = candidates(word)
    candidate_dict = {}
    for item in candidate_words:
        candidate_dict.setdefault(item, 0)
        candidate_dict.update({item : DICTIONARY[item]})

    return max(candidate_dict, key = lambda x : candidate_dict[x])

#!/usr/bin/env python

import string
import os
import sys

DICTIONARY = {}

def read_words():
    '''
    Read bag of words from file
    '''
    f = open('wiktionary', 'r')
    for line in f.readlines():
        DICTIONARY.update({line.split(':')[0] : float(line.split(':')[1])})

def generate_edit_distance1_words(word):
    '''
    Generate a set of all words that are one edit distance from word
    '''
    #delete a character for word
    deletes = [word[1:]]
    deletes += [str(word[:i] + word[i+1:]) for i in range(1, len(word))]
    #change position between two character for word
    transposes = [str(word[1] + word[0] + word[2:])]
    transposes += [str(word[:i-1] + word[i] + word[i-1] + word[i+1:]) for i in range(2, len(word))]
    #replaces one character
    replaces = [str(c + word[1:]) for c in string.lowercase]
    replaces += [str(word[:i] + c + word[i+1:]) for i in range(1, len(word)) for c in string.lowercase]
    #insert one character
    inserts = [str(c + word) for c in string.lowercase]
    inserts += [str(word[:i] + c + word[i:]) for i in range(1, len(word)) for c in string.lowercase]
    inserts += [str(word + c) for c in string.lowercase]

    return set(deletes + transposes + replaces +inserts)

def words_filter(words):
    '''
    Word filter
    '''
    return set(word for word in words if word in DICTIONARY.keys())

def candidates(word):
    '''
    Get all candidates for word
    '''
    if word in DICTIONARY.keys():
        return set([word])
    else:
        return words_filter([word]) | words_filter(generate_edit_distance1_words(word))

def correct(word):
    '''
    correct the word.
    '''
    candidate_words = candidates(word)
    candidate_dict = {}
    for item in candidate_words:
        candidate_dict.setdefault(item, 0)
        candidate_dict.update({item : DICTIONARY[item]})

    return max(candidate_dict, key = lambda x : candidate_dict[x])

if __name__ == '__main__':
    read_words()
    print sys.argv[1], '->', correct(sys.argv[1])

bin/hadoop namenode -format
然后启动Hadoop所有服务，输入命令