Trie 壓縮與文字處理技術詳解

class TrieNode:
    def __init__(self):
        self.children = {}
        self.is_end_of_word = False

def compress_text(text, trie):
    """使用Trie壓縮文字"""
    words = text.split()
    compressed = []
    for word in words:
        node = trie.root
        prefix = ""
        for char in word:
            if char not in node.children:
                break
            prefix += char
            node = node.children[char]
            if node.is_end_of_word:
                break
        compressed.append(prefix)
    return ' '.join(compressed)

# 測試範例
class Trie:
    def __init__(self):
        self.root = TrieNode()

    def insert(self, word):
        node = self.root
        for char in word:
            if char not in node.children:
                node.children[char] = TrieNode()
            node = node.children[char]
        node.is_end_of_word = True

trie = Trie()
trie.insert("apple")
trie.insert("application")
print(compress_text("apple application", trie))

import textwrap

def justify_text(text, width):
    """斷行並調整文字"""
    lines = textwrap.wrap(text, width)
    justified_lines = []
    for line in lines:
        words = line.split()
        if len(words) == 1:
            justified_lines.append(line + ' ' * (width - len(line)))
        else:
            total_chars = sum(len(word) for word in words)
            total_spaces = width - total_chars
            gaps = len(words) - 1
            base_spaces = total_spaces // gaps
            extra_spaces = total_spaces % gaps
            justified_line = ''
            for i, word in enumerate(words):
                justified_line += word
                if i < gaps:
                    justified_line += ' ' * (base_spaces + (1 if i < extra_spaces else 0))
            justified_lines.append(justified_line)
    return '\n'.join(justified_lines)

# 測試範例
text = "This is a sample text that needs to be justified."
print(justify_text(text, 20))

from collections import Counter
import re

def extractive_summarization(text, n_sentences):
    """萃取式摘要"""
    sentences = re.split(r'[.!?]', text)
    sentences = [s.strip() for s in sentences if s.strip()]
    
    word_freq = Counter(' '.join(sentences).split())
    sentence_scores = {}
    for sentence in sentences:
        score = sum(word_freq[word] for word in sentence.split())
        sentence_scores[sentence] = score
    
    top_sentences = sorted(sentence_scores, key=sentence_scores.get, reverse=True)[:n_sentences]
    return ' '.join(top_sentences)

# 測試範例
text = "This is a sample text. It contains multiple sentences. Some sentences are more important than others."
print(extractive_summarization(text, 2))

from nltk.tag import NgramTagger

class CustomNgramTagger(NgramTagger):
    def __init__(self, n, train, backoff=None):
        super().__init__(n, train, backoff=backoff)

    def context_vocab(self, tagged_data):
        """計算上下文詞彙表"""
        # 簡化範例，實際實作需根據需求定義上下文詞彙表的計算邏輯
        vocab = set()
        for sentence in tagged_data:
            for word, tag in sentence:
                vocab.add(word)
        return vocab

# 測試範例
train_data = [[('This', 'DT'), ('is', 'VBZ'), ('a', 'DT'), ('test', 'NN')]]
tagger = CustomNgramTagger(2, train_data)
print(tagger.context_vocab(train_data))

import networkx as nx
import matplotlib.pyplot as plt

def visualize_adjective_network(adjectives, orientations):
    """視覺化形容詞網路"""
    G = nx.Graph()
    for adj in adjectives:
        G.add_node(adj)
    
    # 簡化範例，假設相同取向的形容詞之間有邊
    for i in range(len(adjectives)):
        for j in range(i+1, len(adjectives)):
            if orientations[i] == orientations[j]:
                G.add_edge(adjectives[i], adjectives[j])
    
    nx.draw(G, with_labels=True)
    plt.show()

# 測試範例
adjectives = ["good", "bad", "excellent", "poor"]
orientations = ["positive", "negative", "positive", "negative"]
visualize_adjective_network(adjectives, orientations)

from collections import Counter
import re

def statistically_improbable_phrases(documents, n):
    """發現統計上不常見的短語"""
    # 簡化範例，實際實作需更複雜的邏輯
    all_phrases = []
    for doc in documents:
        phrases = re.findall(r'\b\w+\s\w+\b', doc)
        all_phrases.extend(phrases)
    
    phrase_freq = Counter(all_phrases)
    background_freq = Counter()
    for doc in documents:
        words = doc.split()
        for i in range(len(words)-1):
            background_freq[words[i] + ' ' + words[i+1]] += 1
    
    improbable_phrases = []
    for phrase, freq in phrase_freq.items():
        background_count = background_freq[phrase]
        if freq > background_count * 2:  # 簡化閾值判斷
            improbable_phrases.append((phrase, freq))
    
    return sorted(improbable_phrases, key=lambda x: x[1], reverse=True)[:n]

# 測試範例
documents = ["This is a sample document.", "Another document for testing."]
print(statistically_improbable_phrases(documents, 2))

import itertools

def word_squares(words, n):
    """暴力法尋找詞彙方塊"""
    # 簡化範例，實際實作需更高效的演算法
    for square in itertools.product(words, repeat=n):
        if all(square[i][j] == square[j][i] for i in range(n) for j in range(n)):
            yield square

# 測試範例
words = ["cat", "dog", "sun", "hat"]
for square in word_squares(words, 2):
    for word in square:
        print(word)
    print()

import nltk
text = nltk.word_tokenize("And now for something completely different")
nltk.pos_tag(text)

[('And', 'CC'), ('now', 'RB'), ('for', 'IN'), ('something', 'NN'), 
('completely', 'RB'), ('different', 'JJ')]

tagged_token = nltk.tag.str2tuple('fly/NN')
print(tagged_token)  # ('fly', 'NN')

import nltk
from nltk import pos_tag, word_tokenize

def simple_tagger(sentence):
    tokens = word_tokenize(sentence)
    return pos_tag(tokens)

# 示例用法
sentence = "The quick brown fox jumps over the lazy dog."
print(simple_tagger(sentence))

import nltk
from nltk.metrics import accuracy

# 假設我們有一個標記好的測試集
tagged_test_data = [...]
predicted_tags = [tagger.tag(tokens) for tokens, _ in tagged_test_data]

# 計算準確率
accuracy_score = accuracy(gold_tags, predicted_tags)
print(f"標記準確率：{accuracy_score:.4f}")