自然语言处理

"""中文分词：正向最大匹配 FMM 与逆向最大匹配 BMM"""

DICT = {"研究", "研究生", "生命", "科学", "起源", "的", "自然语言处理",
        "人工智能", "大学", "学生"}
MAX_LEN = 5          # 词典中最长词的字数

def fmm(text):
    """正向最大匹配：从左向右扫描。"""
    result, i = [], 0
    while i < len(text):
        length = min(MAX_LEN, len(text) - i)
        for L in range(length, 0, -1):
            word = text[i:i + L]
            if word in DICT or L == 1:      # 单字兜底
                result.append(word)
                i += L
                break
    return result

def bmm(text):
    """逆向最大匹配：从右向左扫描，通常对汉语更准确。"""
    result, i = [], len(text)
    while i > 0:
        length = min(MAX_LEN, i)
        for L in range(length, 0, -1):
            word = text[i - L:i]
            if word in DICT or L == 1:
                result.append(word)
                i -= L
                break
    return result[::-1]

if __name__ == "__main__":
    for sentence in ["研究生命的起源", "大学生研究人工智能"]:
        f, b = fmm(sentence), bmm(sentence)
        print(f"原句　：{sentence}")
        print(f"FMM 　：{' / '.join(f)}")
        print(f"BMM 　：{' / '.join(b)}")
        print(f"两种切分{'一致' if f == b else '不一致（存在切分歧义）'}\n")

"""英文文本预处理：分句、分词、停用词过滤、简单词干提取"""
import re

STOPWORDS = {"a", "an", "the", "is", "are", "was", "to", "of",
             "and", "in", "on", "it", "this", "that"}

def tokenize(text):
    text = text.lower()
    words = re.findall(r"[a-z]+", text)          # 正则提取单词
    return [w for w in words if w not in STOPWORDS]

def simple_stem(word):
    """极简后缀去除（教学用；生产中用 NLTK 的 PorterStemmer）。"""
    for suf in ["ing", "edly", "ed", "ly", "s"]:
        if word.endswith(suf) and len(word) - len(suf) >= 3:
            return word[:-len(suf)]
    return word

def bag_of_words(text):
    from collections import Counter
    return Counter(simple_stem(w) for w in tokenize(text))

if __name__ == "__main__":
    doc = ("Natural language processing is a subfield of AI. "
           "It studies how computers process human languages.")
    print("分词结果：", tokenize(doc))
    print("词袋统计：", dict(bag_of_words(doc)))

"""TF-IDF 向量化 + 余弦相似度：迷你搜索引擎"""
import math
import re
from collections import Counter

def words(text):
    return re.findall(r"[\u4e00-\u9fa5]{2,}|[a-z]+", text.lower())

class TfidfVectorizer:
    def fit_transform(self, docs):
        tokenized = [words(d) for d in docs]
        self.vocab = sorted({w for d in tokenized for w in d})
        N = len(docs)
        df = Counter()
        for d in tokenized:
            for w in set(d):
                df[w] += 1
        self.idf = {w: math.log(N / (1 + df[w])) + 1 for w in self.vocab}

        vectors = []
        for d in tokenized:
            tf = Counter(d)
            vec = [tf.get(w, 0) * self.idf[w] for w in self.vocab]
            vectors.append(vec)
        return vectors

def cosine(a, b):
    dot = sum(x * y for x, y in zip(a, b))
    na = math.sqrt(sum(x * x for x in a))
    nb = math.sqrt(sum(y * y for y in b))
    return dot / (na * nb + 1e-12)

if __name__ == "__main__":
    docs = [
        "人工智能研究如何让机器感知与推理",
        "机器学习是人工智能的核心技术之一",
        "自然语言处理让计算机理解人类语言",
        "神经网络是机器学习的重要模型",
        "今天天气晴朗适合户外运动",
    ]
    vzer = TfidfVectorizer()
    vectors = vzer.fit_transform(docs)

    query = "人工智能 机器学习"
    q_vec = [words(query).count(w) * vzer.idf[w] if w in vzer.idf else 0
             for w in vzer.vocab]

    scores = sorted(((cosine(q_vec, v), i) for i, v in enumerate(vectors)),
                    reverse=True)
    print(f"查询：{query!r}\n按相关度排序：")
    for score, i in scores:
        print(f"  {score:.3f}  文档{i+1}：{docs[i]}")

"""Bigram 语言模型：句子概率 + 贪心生成"""
from collections import defaultdict

corpus = ["猫 追 老鼠", "狗 追 猫", "猫 爬 树", "狗 叫", "老鼠 跑"]

BOS, EOS = "<s>", "</s>"
bigram = defaultdict(lambda: defaultdict(int))
unigram = defaultdict(int)

for line in corpus:
    toks = [BOS] + line.split() + [EOS]
    for a, b in zip(toks, toks[1:]):
        bigram[a][b] += 1
        unigram[a] += 1

def P(word, prev):
    # 加一平滑，避免零概率
    return (bigram[prev][word] + 1) / (unigram[prev] + len(unigram))

def sentence_prob(sentence):
    toks = [BOS] + sentence.split() + [EOS]
    p = 1.0
    for a, b in zip(toks, toks[1:]):
        p *= P(b, a)
    return p

def generate():
    """贪心解码：每步选条件概率最大的词。"""
    out, cur = [], BOS
    while cur != EOS:
        nxt = max(bigram[cur], key=lambda w: bigram[cur][w])
        if nxt == EOS:
            break
        out.append(nxt)
        cur = nxt
    return " ".join(out)

if __name__ == "__main__":
    s1 = "猫 追 老鼠"
    s2 = "老鼠 追 狗"
    print(f"P({s1!r}) = {sentence_prob(s1):.5f}")
    print(f"P({s2!r}) = {sentence_prob(s2):.5f}  （罕见组合，概率更低）")
    print("贪心生成句子：", generate())

"""电影评论情感二分类：多项式朴素贝叶斯（拉普拉斯平滑）"""
from collections import defaultdict
import math

train_data = [
    ("这部 电影 剧情 精彩 演员 表演 出色", "正面"),
    ("非常 喜欢 这部 电影 很 精彩", "正面"),
    ("剧情 无聊 表演 糟糕 浪费 时间", "负面"),
    ("电影 太 糟糕 了 非常 失望", "负面"),
]

class MultinomialNBText:
    def fit(self, data):
        self.wc = defaultdict(lambda: defaultdict(int))
        self.doc_count = defaultdict(int)
        vocab = set()
        for text, label in data:
            self.doc_count[label] += 1
            for w in text.split():
                self.wc[label][w] += 1
                vocab.add(w)
        self.V = len(vocab)
        self.total = {c: sum(self.wc[c].values()) for c in self.doc_count}
        self.N = len(data)

    def predict(self, text):
        scores = {}
        for c in self.doc_count:
            log_p = math.log(self.doc_count[c] / self.N)       # 先验
            for w in text.split():
                tf = self.wc[c][w] + 1                         # 拉普拉斯平滑
                log_p += math.log(tf / (self.total[c] + self.V))
            scores[c] = math.exp(log_p)
        z = sum(scores.values())
        return sorted(((c, p / z) for c, p in scores.items()),
                      key=lambda kv: -kv[1])

if __name__ == "__main__":
    model = MultinomialNBText()
    model.fit(train_data)
    for text in ["剧情 精彩 表演 出色", "剧情 无聊 很 失望"]:
        probs = model.predict(text)
        pred, conf = probs[0]
        detail = "，".join(f"{c}:{p:.3f}" for c, p in probs)
        print(f"评论：{text}\n  -> {pred}（置信度 {conf:.2f}）[{detail}]\n")

"""基于后缀规则的中文命名实体识别（教学版）"""

SUFFIX = {
    "ORG": ["大学", "公司", "医院", "银行", "研究院"],
    "LOC": ["市", "省", "县", "自治区"],
}
# 标点与常见动词视为实体的左边界
BOUNDARY = set("，。；：、 ,; 位在于到")

def find_entities(text):
    entities, seen = [], set()
    for label, suffixes in SUFFIX.items():
        for suf in suffixes:
            start_search = 0
            while True:
                idx = text.find(suf, start_search)
                if idx == -1:
                    break
                end = idx + len(suf)
                # 从后缀向左找最近边界
                left = idx - 1
                while left >= 0 and text[left] not in BOUNDARY and end - left <= 6:
                    left -= 1
                entity = text[left + 1:end]
                if entity and entity not in seen:
                    seen.add(entity)
                    entities.append((entity, label))
                start_search = end
    return entities

if __name__ == "__main__":
    text = "北京大学位于北京市，华为公司总部在深圳市。"
    print("原文：", text)
    for entity, label in find_entities(text):
        label_zh = {"ORG": "机构名", "LOC": "地名"}[label]
        print(f"  {entity}　——>　{label_zh}")

"""基于情感词典的情感分析（支持否定反转、程度副词加强）"""

LEXICON = {"好": 1, "优秀": 1, "喜欢": 1, "精彩": 1, "满意": 1,
           "糟糕": -1, "差": -1, "失望": -1, "讨厌": -1, "无聊": -1}
NEGATION = {"不", "没", "没有", "别", "不是"}
DEGREE = {"很": 1.5, "非常": 2.0, "特别": 1.8, "有点": 0.5}

def sentiment_score(text):
    score = 0.0
    neg, degree = False, 1.0
    for w in text.split():
        if w in NEGATION:
            neg = True                       # 否定作用于下一个情感词
        elif w in DEGREE:
            degree = DEGREE[w]
        elif w in LEXICON:
            s = LEXICON[w] * degree * (-1 if neg else 1)
            score += s
            neg, degree = False, 1.0         # 计分后复位
    return score

def polarity(score):
    if score > 0.2:  return "正面"
    if score < -0.2: return "负面"
    return "中性"

if __name__ == "__main__":
    reviews = [
        "我 不 喜欢 这部 很 糟糕 的 电影",
        "这部 电影 很 精彩 我 非常 喜欢",
        "电影 时长 两个 小时",
    ]
    for r in reviews:
        s = sentiment_score(r)
        print(f"得分 {s:+.1f}　{polarity(s)}　｜ {r.replace(' ', '')}")

"""使用 HuggingFace Transformers 调用现代中文模型
   需先安装：pip install transformers torch
   首次运行会自动下载模型（需要联网）。"""
try:
    from transformers import pipeline

    # 情感分析：一行代码加载预训练模型
    clf = pipeline("sentiment-analysis",
                   model="uer/roberta-base-finetuned-dianping-chinese")
    print(clf("这家餐厅环境优雅，菜品精致，服务也非常周到。"))
    # 输出形如：[{'label': 'positive (stars 5)', 'score': 0.98}]

    # 文本生成：提示词驱动
    gen = pipeline("text-generation", model="Qwen/Qwen2.5-0.5B")
    print(gen("用三句话解释什么是人工智能：", max_new_tokens=120))
except ImportError:
    print("请先安装 transformers 与 torch，再运行本例。")