/* * Vocabulary Learning Curve Analyzer * * For each excerpt length (1, 2, 3, ... N words), finds the excerpt that * requires the minimum number of top-frequency words to understand 100%. * * Usage: * ./vocabulary_curve [max_length] * ./vocabulary_curve test.txt 50 */ #include #include #include #include #include #define MAX_WORD_LEN 64 #define MAX_WORDS 500000 #define MAX_UNIQUE_WORDS 100000 #define HASH_SIZE 200003 /* Prime number for better distribution */ /* Word entry for hash table */ typedef struct WordEntry { char word[MAX_WORD_LEN]; int count; int rank; /* 1-indexed rank by frequency (1 = most common) */ struct WordEntry *next; } WordEntry; /* Hash table for word lookup */ static WordEntry *hash_table[HASH_SIZE]; static WordEntry *all_entries[MAX_UNIQUE_WORDS]; static int num_unique_words = 0; /* All words in order of appearance - store POINTERS not indices */ static WordEntry *word_sequence[MAX_WORDS]; static int num_words = 0; /* Result for each excerpt length */ typedef struct { int excerpt_length; int min_vocab_needed; int start_pos; /* Start position in word_sequence */ } ExcerptResult; /* Simple hash function */ static unsigned int hash_word(const char *word) { unsigned int hash = 5381; int c; while ((c = *word++)) { hash = ((hash << 5) + hash) + c; } return hash % HASH_SIZE; } /* Find or create word entry */ static WordEntry *get_or_create_word(const char *word) { unsigned int h = hash_word(word); WordEntry *entry = hash_table[h]; while (entry) { if (strcmp(entry->word, word) == 0) { return entry; } entry = entry->next; } /* Create new entry */ if (num_unique_words >= MAX_UNIQUE_WORDS) { fprintf(stderr, "Too many unique words\n"); exit(1); } entry = malloc(sizeof(WordEntry)); if (!entry) { fprintf(stderr, "Memory allocation failed\n"); exit(1); } strncpy(entry->word, word, MAX_WORD_LEN - 1); entry->word[MAX_WORD_LEN - 1] = '\0'; entry->count = 0; entry->rank = 0; entry->next = hash_table[h]; hash_table[h] = entry; all_entries[num_unique_words++] = entry; return entry; } /* Compare function for sorting by frequency (descending) */ static int compare_by_count(const void *a, const void *b) { const WordEntry *wa = *(const WordEntry **)a; const WordEntry *wb = *(const WordEntry **)b; return wb->count - wa->count; /* Descending */ } /* Check if character is part of a word */ static bool is_word_char(int c) { return isalnum(c) || c == '_' || (unsigned char)c >= 128; } /* Read and process file */ static bool process_file(const char *filename) { FILE *fp = fopen(filename, "r"); if (!fp) { fprintf(stderr, "Cannot open file: %s\n", filename); return false; } char word[MAX_WORD_LEN]; int word_len = 0; int c; while ((c = fgetc(fp)) != EOF) { if (is_word_char(c)) { if (word_len < MAX_WORD_LEN - 1) { word[word_len++] = tolower(c); } } else if (word_len > 0) { word[word_len] = '\0'; WordEntry *entry = get_or_create_word(word); entry->count++; if (num_words >= MAX_WORDS) { fprintf(stderr, "Too many words in file\n"); fclose(fp); return false; } /* Store pointer directly - survives sorting */ word_sequence[num_words++] = entry; word_len = 0; } } /* Handle last word if file doesn't end with whitespace */ if (word_len > 0) { word[word_len] = '\0'; WordEntry *entry = get_or_create_word(word); entry->count++; if (num_words < MAX_WORDS) { word_sequence[num_words++] = entry; } } fclose(fp); return true; } /* Assign ranks based on frequency */ static void assign_ranks(void) { /* Sort all_entries by frequency (this doesn't affect word_sequence) */ qsort(all_entries, num_unique_words, sizeof(WordEntry *), compare_by_count); /* Assign 1-indexed ranks using competition ranking: * Words with same frequency get same rank. * Next rank is current_position + 1 (skipping numbers). * Example: counts 5,3,3,2 -> ranks 1,2,2,4 (not 1,2,3,4) */ for (int i = 0; i < num_unique_words; i++) { if (i == 0) { all_entries[i]->rank = 1; } else if (all_entries[i]->count == all_entries[i - 1]->count) { /* Same frequency as previous word - same rank */ all_entries[i]->rank = all_entries[i - 1]->rank; } else { /* Different frequency - rank is position + 1 */ all_entries[i]->rank = i + 1; } } } /* Analyze excerpt and return max rank needed */ static int analyze_excerpt(int start, int length) { /* Track which entries we've seen using a simple visited array */ /* We use the rank field is already assigned, so we can check uniqueness */ static bool seen_rank[MAX_UNIQUE_WORDS + 1]; memset(seen_rank, 0, (num_unique_words + 1) * sizeof(bool)); int max_rank = 0; for (int i = start; i < start + length; i++) { WordEntry *entry = word_sequence[i]; int rank = entry->rank; if (!seen_rank[rank]) { seen_rank[rank] = true; if (rank > max_rank) { max_rank = rank; } } } return max_rank; } /* Find optimal excerpts for each length */ static void find_optimal_excerpts(int max_length, ExcerptResult *results) { for (int length = 1; length <= max_length && length <= num_words; length++) { int best_vocab = num_unique_words + 1; int best_start = 0; /* Slide window through text */ for (int start = 0; start <= num_words - length; start++) { int vocab_needed = analyze_excerpt(start, length); if (vocab_needed < best_vocab) { best_vocab = vocab_needed; best_start = start; } } results[length - 1].excerpt_length = length; results[length - 1].min_vocab_needed = best_vocab; results[length - 1].start_pos = best_start; } } /* Print excerpt words */ static void print_excerpt(int start, int length) { for (int i = start; i < start + length; i++) { if (i > start) printf(" "); printf("%s", word_sequence[i]->word); } } /* Print words needed (sorted by rank) */ static void print_words_needed(int start, int length) { /* Collect unique entries */ static WordEntry *unique_entries[MAX_UNIQUE_WORDS]; static bool seen_rank[MAX_UNIQUE_WORDS + 1]; memset(seen_rank, 0, (num_unique_words + 1) * sizeof(bool)); int count = 0; for (int i = start; i < start + length; i++) { WordEntry *entry = word_sequence[i]; if (!seen_rank[entry->rank]) { seen_rank[entry->rank] = true; unique_entries[count++] = entry; } } /* Sort by rank (simple bubble sort - small arrays) */ for (int i = 0; i < count - 1; i++) { for (int j = i + 1; j < count; j++) { if (unique_entries[i]->rank > unique_entries[j]->rank) { WordEntry *tmp = unique_entries[i]; unique_entries[i] = unique_entries[j]; unique_entries[j] = tmp; } } } /* Print */ for (int i = 0; i < count; i++) { if (i > 0) printf(", "); printf("%s(#%d)", unique_entries[i]->word, unique_entries[i]->rank); } } /* Print results */ static void print_results(ExcerptResult *results, int max_length) { printf("======================================================================\n"); printf("VOCABULARY LEARNING CURVE\n"); printf("======================================================================\n"); printf("\n"); printf("For each excerpt length, the minimum number of top-frequency\n"); printf("words you need to learn to understand 100%% of some excerpt.\n"); printf("\n"); printf("Total words in text: %d\n", num_words); printf("Unique words: %d\n", num_unique_words); printf("\n"); printf("----------------------------------------------------------------------\n"); int prev_vocab = 0; int actual_max = max_length; if (actual_max > num_words) actual_max = num_words; for (int i = 0; i < actual_max; i++) { ExcerptResult *r = &results[i]; printf("\n[Length %d] Vocab needed: %d", r->excerpt_length, r->min_vocab_needed); if (r->min_vocab_needed > prev_vocab) { printf(" (+%d)", r->min_vocab_needed - prev_vocab); } printf("\n"); printf(" Excerpt: \""); print_excerpt(r->start_pos, r->excerpt_length); printf("\"\n"); printf(" Words: "); print_words_needed(r->start_pos, r->excerpt_length); printf("\n"); prev_vocab = r->min_vocab_needed; } printf("\n----------------------------------------------------------------------\n"); if (actual_max > 0) { ExcerptResult *final = &results[actual_max - 1]; printf("\nTo understand a %d-word excerpt,\n", final->excerpt_length); printf("you need to learn at minimum %d top words.\n", final->min_vocab_needed); } } /* Free memory */ static void cleanup(void) { for (int i = 0; i < num_unique_words; i++) { free(all_entries[i]); } } /* Dump all vocabulary with ranks (for Python integration) */ static void dump_vocabulary(int max_rank) { printf("VOCAB_DUMP_START\n"); for (int i = 0; i < num_unique_words; i++) { if (all_entries[i]->rank <= max_rank) { printf("%s;%d\n", all_entries[i]->word, all_entries[i]->rank); } } printf("VOCAB_DUMP_END\n"); } /* Find longest excerpt using only top N words (inverse mode) */ static void find_longest_excerpt(int max_vocab) { /* Sliding window: find longest contiguous sequence where all words have rank <= max_vocab */ int best_start = 0; int best_length = 0; int left = 0; for (int right = 0; right < num_words; right++) { /* If current word is outside our vocabulary, move left past it */ if (word_sequence[right]->rank > max_vocab) { left = right + 1; } else { /* Current window [left, right] is valid */ int length = right - left + 1; if (length > best_length) { best_length = length; best_start = left; } } } /* Print results */ printf("======================================================================\n"); printf("INVERSE MODE: LONGEST EXCERPT WITH TOP %d WORDS\n", max_vocab); printf("======================================================================\n"); printf("\n"); printf("Total words in text: %d\n", num_words); printf("Unique words: %d\n", num_unique_words); printf("Vocabulary limit: top %d words\n", max_vocab); printf("\n"); printf("----------------------------------------------------------------------\n"); printf("\n"); if (best_length == 0) { printf("No valid excerpt found with top %d words.\n", max_vocab); printf("The text may require rarer words from the very beginning.\n"); } else { printf("LONGEST EXCERPT: %d words\n", best_length); printf("Position: words %d to %d\n", best_start + 1, best_start + best_length); printf("\n"); printf("Excerpt:\n \""); print_excerpt(best_start, best_length); printf("\"\n"); printf("\n"); /* Find the rarest word in the excerpt */ int max_rank_used = 0; const char *rarest_word = NULL; for (int i = best_start; i < best_start + best_length; i++) { if (word_sequence[i]->rank > max_rank_used) { max_rank_used = word_sequence[i]->rank; rarest_word = word_sequence[i]->word; } } // cppcheck-suppress nullPointer printf("Rarest word used: %s (#%d)\n", rarest_word, max_rank_used); /* Count unique words in excerpt */ static bool seen_rank[MAX_UNIQUE_WORDS + 1]; memset(seen_rank, 0, (num_unique_words + 1) * sizeof(bool)); int unique_count = 0; for (int i = best_start; i < best_start + best_length; i++) { if (!seen_rank[word_sequence[i]->rank]) { seen_rank[word_sequence[i]->rank] = true; unique_count++; } } printf("Unique words in excerpt: %d\n", unique_count); } printf("\n----------------------------------------------------------------------\n"); } int main(int argc, char *argv[]) { if (argc < 2) { fprintf(stderr, "Usage: %s [options]\n", argv[0]); fprintf(stderr, "\nModes:\n"); fprintf(stderr, " (default) Find minimum vocab needed for each excerpt length\n"); fprintf(stderr, " --max-vocab N INVERSE: Find longest excerpt using only top N words\n"); fprintf(stderr, "\nOptions:\n"); fprintf(stderr, " max_length Maximum excerpt length to analyze (default: 30)\n"); fprintf(stderr, " --dump-vocab [N] Output all words with ranks up to N\n"); fprintf(stderr, "\nExamples:\n"); fprintf(stderr, " %s book.txt 50 # Analyze excerpts up to 50 words\n", argv[0]); fprintf(stderr, " %s book.txt --max-vocab 500 # Find longest excerpt with top 500 words\n", argv[0]); return 1; } const char *filename = argv[1]; int max_length = 30; bool dump_vocab = false; int dump_max_rank = 0; int max_vocab_mode = 0; /* 0 = normal mode, >0 = inverse mode with this vocab limit */ /* Parse arguments */ for (int i = 2; i < argc; i++) { if (strcmp(argv[i], "--dump-vocab") == 0) { dump_vocab = true; if (i + 1 < argc && argv[i + 1][0] != '-') { dump_max_rank = atoi(argv[++i]); } } else if (strcmp(argv[i], "--max-vocab") == 0) { if (i + 1 < argc) { max_vocab_mode = atoi(argv[++i]); if (max_vocab_mode < 1) { fprintf(stderr, "Error: --max-vocab requires a positive number\n"); return 1; } } else { fprintf(stderr, "Error: --max-vocab requires a number\n"); return 1; } } else if (argv[i][0] != '-') { max_length = atoi(argv[i]); if (max_length < 1) max_length = 1; if (max_length > 1000) max_length = 1000; } } /* Initialize hash table */ memset(hash_table, 0, sizeof(hash_table)); /* Process file */ if (!process_file(filename)) { return 1; } if (num_words == 0) { fprintf(stderr, "No words found in file\n"); return 1; } /* Assign ranks by frequency */ assign_ranks(); /* Inverse mode: find longest excerpt with limited vocabulary */ if (max_vocab_mode > 0) { find_longest_excerpt(max_vocab_mode); /* Dump vocabulary if requested */ if (dump_vocab) { if (dump_max_rank == 0) dump_max_rank = max_vocab_mode; dump_vocabulary(dump_max_rank); } cleanup(); return 0; } /* Normal mode: find optimal excerpts */ ExcerptResult *results = malloc(max_length * sizeof(ExcerptResult)); if (!results) { fprintf(stderr, "Memory allocation failed\n"); cleanup(); return 1; } find_optimal_excerpts(max_length, results); /* Print results */ print_results(results, max_length); /* Dump vocabulary if requested */ if (dump_vocab) { /* If no max_rank specified, use the max from the excerpt */ if (dump_max_rank == 0 && max_length > 0) { dump_max_rank = results[max_length - 1].min_vocab_needed; } if (dump_max_rank > 0) { dump_vocabulary(dump_max_rank); } } /* Cleanup */ free(results); cleanup(); return 0; }