mirror of
https://github.com/kuhyx/WUT_Computer_Science.git
synced 2026-11-20 15:32:24 +01:00
feat: split gpt chunks into smaller function and blacked it
This commit is contained in:
parent
442d5c4209
commit
bc88b74648
349
gpt_chunks.py
349
gpt_chunks.py
@ -2,22 +2,25 @@ from openai import OpenAI
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
|
||||||
def sentence_loop(sentences):
|
def sentence_loop(sentences):
|
||||||
# Process each sentence
|
# Process each sentence
|
||||||
chunked_sentences = []
|
chunked_sentences = []
|
||||||
for sentence in sentences:
|
for sentence in sentences:
|
||||||
chunked_sentences = process_sentence(sentence, chunked_sentences)
|
chunked_sentences = process_sentence(sentence, chunked_sentences)
|
||||||
if chunked_sentences == False:
|
if chunked_sentences == False:
|
||||||
return
|
return
|
||||||
return chunked_sentences
|
return chunked_sentences
|
||||||
|
|
||||||
|
|
||||||
def is_correct_format(s):
|
def is_correct_format(s):
|
||||||
# Regular expression to match sequences of '[ some words ]'
|
# Regular expression to match sequences of '[ some words ]'
|
||||||
pattern = r'^(\[\s*[^]]+\s*\]\s*)+$'
|
pattern = r"^(\[\s*[^]]+\s*\]\s*)+$"
|
||||||
|
|
||||||
# Check if the string matches the pattern
|
# Check if the string matches the pattern
|
||||||
return bool(re.match(pattern, s))
|
return bool(re.match(pattern, s))
|
||||||
|
|
||||||
|
|
||||||
def check_format(input_data):
|
def check_format(input_data):
|
||||||
# Check if the input is a string, and if so, process it directly
|
# Check if the input is a string, and if so, process it directly
|
||||||
if isinstance(input_data, str):
|
if isinstance(input_data, str):
|
||||||
@ -31,76 +34,84 @@ def check_format(input_data):
|
|||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def remove_leading_numbering(string):
|
||||||
|
return re.sub(r"^\d+\.\s*", "", string)
|
||||||
|
|
||||||
|
|
||||||
|
def process_brackets(string):
|
||||||
|
parts = re.findall(r"\[[^\]]*\]", string)
|
||||||
|
if parts:
|
||||||
|
parts[-1] = handle_trailing_punctuation(string, parts[-1])
|
||||||
|
return parts
|
||||||
|
|
||||||
|
|
||||||
|
def handle_trailing_punctuation(string, last_part):
|
||||||
|
trailing_punctuation = re.search(r"\]\s*([.!?])\s*$", string)
|
||||||
|
if trailing_punctuation:
|
||||||
|
return last_part[:-1] + trailing_punctuation.group(1) + "]"
|
||||||
|
return last_part
|
||||||
|
|
||||||
|
|
||||||
|
def process_non_bracketed_parts(string):
|
||||||
|
parts = re.split(r"\s*-\s*", string)
|
||||||
|
return [f"[ {part.strip()} ]" for part in parts if part.strip()]
|
||||||
|
|
||||||
|
|
||||||
def reformat_list(strings):
|
def reformat_list(strings):
|
||||||
result = []
|
result = []
|
||||||
|
|
||||||
for string in strings:
|
for string in strings:
|
||||||
# Remove leading numbering like '1. '
|
string = remove_leading_numbering(string)
|
||||||
string = re.sub(r'^\d+\.\s*', '', string)
|
if "[" in string and "]" in string:
|
||||||
|
result.extend(process_brackets(string))
|
||||||
# Check if the string contains partially formatted brackets
|
|
||||||
if '[' in string and ']' in string:
|
|
||||||
# Splitting the string into parts that are enclosed in square brackets
|
|
||||||
parts = re.findall(r'\[[^\]]*\]', string)
|
|
||||||
|
|
||||||
# Check for trailing punctuation after the last bracket and add it to the last part
|
|
||||||
trailing_punctuation = re.search(r'\]\s*([.!?])\s*$', string)
|
|
||||||
if trailing_punctuation and parts:
|
|
||||||
parts[-1] = parts[-1][:-1] + trailing_punctuation.group(1) + ']'
|
|
||||||
|
|
||||||
result.extend(parts)
|
|
||||||
|
|
||||||
# Handle strings without square brackets
|
|
||||||
else:
|
else:
|
||||||
# Splitting based on the dashed pattern and removing the numbering
|
result.extend(process_non_bracketed_parts(string))
|
||||||
parts = re.split(r'\s*-\s*', string)
|
return " ".join(result)
|
||||||
parts = [part.strip() for part in parts if part.strip()]
|
|
||||||
|
|
||||||
# Enclosing each part in square brackets
|
|
||||||
for part in parts:
|
|
||||||
result.append(f'[ {part} ]')
|
|
||||||
|
|
||||||
return ' '.join(result)
|
|
||||||
|
|
||||||
def reformat_square_not_full(input_string):
|
def reformat_square_not_full(input_string):
|
||||||
# Split the string into parts that are either in brackets or not
|
# Split the string into parts that are either in brackets or not
|
||||||
parts = re.split(r'(\[[^\]]*\])', input_string)
|
parts = re.split(r"(\[[^\]]*\])", input_string)
|
||||||
|
|
||||||
formatted_parts = []
|
formatted_parts = []
|
||||||
for part in parts:
|
for part in parts:
|
||||||
if part: # Ignore empty strings
|
if part: # Ignore empty strings
|
||||||
# For parts not in brackets, remove trailing period, trim spaces, and enclose in brackets
|
# For parts not in brackets, remove trailing period, trim spaces, and enclose in brackets
|
||||||
if not part.startswith('['):
|
if not part.startswith("["):
|
||||||
part = re.sub(r'\.\s*$', '', part).strip()
|
part = re.sub(r"\.\s*$", "", part).strip()
|
||||||
formatted_parts.append(f"[ {part} ]")
|
formatted_parts.append(f"[ {part} ]")
|
||||||
else:
|
else:
|
||||||
# For parts already in brackets, add spaces inside
|
# For parts already in brackets, add spaces inside
|
||||||
formatted_parts.append(re.sub(r'\[([^]]+)\]', r'[ \1 ]', part))
|
formatted_parts.append(re.sub(r"\[([^]]+)\]", r"[ \1 ]", part))
|
||||||
|
|
||||||
|
return " ".join(formatted_parts)
|
||||||
|
|
||||||
return ' '.join(formatted_parts)
|
|
||||||
|
|
||||||
def reformat_slash(input_string):
|
def reformat_slash(input_string):
|
||||||
# Split the string on slashes and strip spaces
|
# Split the string on slashes and strip spaces
|
||||||
parts = [part.strip() for part in input_string.split('/')]
|
parts = [part.strip() for part in input_string.split("/")]
|
||||||
# Enclose each part in square brackets
|
# Enclose each part in square brackets
|
||||||
formatted_parts = [f"[ {part} ]" for part in parts]
|
formatted_parts = [f"[ {part} ]" for part in parts]
|
||||||
return ' '.join(formatted_parts)
|
return " ".join(formatted_parts)
|
||||||
|
|
||||||
|
|
||||||
def reformat_pipe(input_string):
|
def reformat_pipe(input_string):
|
||||||
# Split the string on ' | ' and strip spaces
|
# Split the string on ' | ' and strip spaces
|
||||||
parts = [part.strip() for part in input_string.split('|')]
|
parts = [part.strip() for part in input_string.split("|")]
|
||||||
# Enclose each part in square brackets
|
# Enclose each part in square brackets
|
||||||
formatted_parts = [f"[ {part} ]" for part in parts]
|
formatted_parts = [f"[ {part} ]" for part in parts]
|
||||||
return ' '.join(formatted_parts)
|
return " ".join(formatted_parts)
|
||||||
|
|
||||||
|
|
||||||
def is_pipe_format(s):
|
def is_pipe_format(s):
|
||||||
# Regular expression to check for ' | ' pattern
|
# Regular expression to check for ' | ' pattern
|
||||||
pattern = r'^(?:[^|]+\|)+[^|]+$'
|
pattern = r"^(?:[^|]+\|)+[^|]+$"
|
||||||
return bool(re.match(pattern, s))
|
return bool(re.match(pattern, s))
|
||||||
|
|
||||||
|
|
||||||
def is_numbered_line_format(s):
|
def is_numbered_line_format(s):
|
||||||
# Regular expression to match lines starting with 'number. word/phrase \n'
|
# Regular expression to match lines starting with 'number. word/phrase \n'
|
||||||
pattern = r'^(?:\d+\.\s+.+\n?)+$'
|
pattern = r"^(?:\d+\.\s+.+\n?)+$"
|
||||||
|
|
||||||
# Check if the input is a list of strings
|
# Check if the input is a list of strings
|
||||||
if isinstance(s, list):
|
if isinstance(s, list):
|
||||||
@ -109,205 +120,239 @@ def is_numbered_line_format(s):
|
|||||||
# If the input is a single string, process it directly
|
# If the input is a single string, process it directly
|
||||||
return bool(re.match(pattern, s))
|
return bool(re.match(pattern, s))
|
||||||
|
|
||||||
|
|
||||||
def reformat_ists(input_string):
|
def reformat_ists(input_string):
|
||||||
# Use regular expression to find all occurrences of the iSTS pattern and extract the text
|
# Use regular expression to find all occurrences of the iSTS pattern and extract the text
|
||||||
parts = re.findall(r'\(iSTS \d+\)\s*([^)]+)', input_string)
|
parts = re.findall(r"\(iSTS \d+\)\s*([^)]+)", input_string)
|
||||||
# Enclose each extracted part in square brackets
|
# Enclose each extracted part in square brackets
|
||||||
formatted_parts = [f"[ {part.strip()} ]" for part in parts]
|
formatted_parts = [f"[ {part.strip()} ]" for part in parts]
|
||||||
return ' '.join(formatted_parts)
|
return " ".join(formatted_parts)
|
||||||
|
|
||||||
|
|
||||||
def is_ists_format(s):
|
def is_ists_format(s):
|
||||||
# Regular expression to check for the iSTS pattern
|
# Regular expression to check for the iSTS pattern
|
||||||
pattern = r'^(?:\(iSTS \d+\)\s*[^)]+\s*)+$'
|
pattern = r"^(?:\(iSTS \d+\)\s*[^)]+\s*)+$"
|
||||||
return bool(re.match(pattern, s))
|
return bool(re.match(pattern, s))
|
||||||
|
|
||||||
|
|
||||||
def reformat_with_sections(input_string):
|
def reformat_with_sections(input_string):
|
||||||
# Use regular expression to extract text following the section labels
|
# Use regular expression to extract text following the section labels
|
||||||
parts = re.split(r'\s*\[\s*S\d+\s*\]\s*', input_string)
|
parts = re.split(r"\s*\[\s*S\d+\s*\]\s*", input_string)
|
||||||
# Remove empty strings and enclose each part in square brackets
|
# Remove empty strings and enclose each part in square brackets
|
||||||
formatted_parts = [f"[ {part.strip()} ]" for part in parts if part.strip()]
|
formatted_parts = [f"[ {part.strip()} ]" for part in parts if part.strip()]
|
||||||
return ' '.join(formatted_parts)
|
return " ".join(formatted_parts)
|
||||||
|
|
||||||
|
|
||||||
def is_section_format(s):
|
def is_section_format(s):
|
||||||
# Regular expression to check for the section label pattern
|
# Regular expression to check for the section label pattern
|
||||||
pattern = r'^(?:\s*\[\s*S\d+\s*\]\s*.+)+$'
|
pattern = r"^(?:\s*\[\s*S\d+\s*\]\s*.+)+$"
|
||||||
return bool(re.match(pattern, s))
|
return bool(re.match(pattern, s))
|
||||||
|
|
||||||
|
|
||||||
def is_chunk_format(s):
|
def is_chunk_format(s):
|
||||||
# Regular expression to check for both 'Chunk X: text' and '[Chunk X] text' patterns
|
# Regular expression to check for both 'Chunk X: text' and '[Chunk X] text' patterns
|
||||||
pattern = r'^(Chunk \d+:\s*.+?|\[\s*Chunk \d+\s*\]\s*.+?)(\s+(Chunk \d+:\s*.+?|\[\s*Chunk \d+\s*\]\s*.+?))*$'
|
pattern = r"^(Chunk \d+:\s*.+?|\[\s*Chunk \d+\s*\]\s*.+?)(\s+(Chunk \d+:\s*.+?|\[\s*Chunk \d+\s*\]\s*.+?))*$"
|
||||||
return bool(re.match(pattern, s))
|
return bool(re.match(pattern, s))
|
||||||
|
|
||||||
|
|
||||||
def reformat_chunks(input_string):
|
def reformat_chunks(input_string):
|
||||||
# Find all occurrences of both chunk patterns and extract the text
|
# Find all occurrences of both chunk patterns and extract the text
|
||||||
parts = re.findall(r'Chunk \d+:\s*(.+?)(?=\s*Chunk \d+:|$)|\[\s*Chunk \d+\s*\]\s*(.+?)(?=\s*\[\s*Chunk \d+\s*\]|$)', input_string)
|
parts = re.findall(
|
||||||
|
r"Chunk \d+:\s*(.+?)(?=\s*Chunk \d+:|$)|\[\s*Chunk \d+\s*\]\s*(.+?)(?=\s*\[\s*Chunk \d+\s*\]|$)",
|
||||||
|
input_string,
|
||||||
|
)
|
||||||
|
|
||||||
# Flatten the list of tuples and remove empty strings
|
# Flatten the list of tuples and remove empty strings
|
||||||
flattened_parts = [item for sublist in parts for item in sublist if item]
|
flattened_parts = [item for sublist in parts for item in sublist if item]
|
||||||
|
|
||||||
# Enclose each extracted part in square brackets
|
# Enclose each extracted part in square brackets
|
||||||
formatted_parts = [f"[ {part.strip()} ]" for part in flattened_parts]
|
formatted_parts = [f"[ {part.strip()} ]" for part in flattened_parts]
|
||||||
return ' '.join(formatted_parts)
|
return " ".join(formatted_parts)
|
||||||
|
|
||||||
|
|
||||||
def reformat_ists_markers(input_string):
|
def reformat_ists_markers(input_string):
|
||||||
# Split the string at '[ iSTS ]', remove empty strings, and enclose each part in square brackets
|
# Split the string at '[ iSTS ]', remove empty strings, and enclose each part in square brackets
|
||||||
parts = [part.strip() for part in re.split(r'\[\s*iSTS\s*\]', input_string) if part.strip()]
|
parts = [
|
||||||
|
part.strip()
|
||||||
|
for part in re.split(r"\[\s*iSTS\s*\]", input_string)
|
||||||
|
if part.strip()
|
||||||
|
]
|
||||||
formatted_parts = [f"[ {part} ]" for part in parts]
|
formatted_parts = [f"[ {part} ]" for part in parts]
|
||||||
return ' '.join(formatted_parts)
|
return " ".join(formatted_parts)
|
||||||
|
|
||||||
|
|
||||||
def is_ists_marker_format(s):
|
def is_ists_marker_format(s):
|
||||||
# Regular expression to check for the pattern with '[ iSTS ]'
|
# Regular expression to check for the pattern with '[ iSTS ]'
|
||||||
pattern = r'^(?:\[\s*iSTS\s*\].+)+$'
|
pattern = r"^(?:\[\s*iSTS\s*\].+)+$"
|
||||||
return bool(re.match(pattern, s))
|
return bool(re.match(pattern, s))
|
||||||
|
|
||||||
|
|
||||||
def reformat_with_dashes(input_string):
|
def reformat_with_dashes(input_string):
|
||||||
# Split the string on ' - ', remove empty strings, and enclose each part in square brackets
|
# Split the string on ' - ', remove empty strings, and enclose each part in square brackets
|
||||||
parts = [part.strip() for part in input_string.split('-') if part.strip()]
|
parts = [part.strip() for part in input_string.split("-") if part.strip()]
|
||||||
formatted_parts = [f"[ {part} ]" for part in parts]
|
formatted_parts = [f"[ {part} ]" for part in parts]
|
||||||
return ' '.join(formatted_parts)
|
return " ".join(formatted_parts)
|
||||||
|
|
||||||
|
|
||||||
def is_dash_format(s):
|
def is_dash_format(s):
|
||||||
# Regular expression to check for the pattern with ' - '
|
# Regular expression to check for the pattern with ' - '
|
||||||
pattern = r'^[^-]+(?:\s*-\s*[^-]+)+$'
|
pattern = r"^[^-]+(?:\s*-\s*[^-]+)+$"
|
||||||
return bool(re.match(pattern, s))
|
return bool(re.match(pattern, s))
|
||||||
|
|
||||||
|
|
||||||
def preliminary_reformat(input_string):
|
def preliminary_reformat(input_string):
|
||||||
input_string = input_string.strip("iSTS chunks:")
|
input_string = input_string.strip("iSTS chunks:")
|
||||||
input_string = input_string.strip("Here are the iSTS chunks for the given sentence:")
|
input_string = input_string.strip(
|
||||||
return input_string
|
"Here are the iSTS chunks for the given sentence:"
|
||||||
|
)
|
||||||
|
return input_string
|
||||||
|
|
||||||
|
|
||||||
def reformat_brackets_and_text(input_string):
|
def reformat_brackets_and_text(input_string):
|
||||||
# Remove the trailing dot if present
|
# Remove the trailing dot if present
|
||||||
input_string = re.sub(r'\.$', '', input_string)
|
input_string = re.sub(r"\.$", "", input_string)
|
||||||
|
|
||||||
# Split the string into parts that are either in brackets or not
|
# Split the string into parts that are either in brackets or not
|
||||||
parts = re.split(r'(\[[^\]]*\])', input_string)
|
parts = re.split(r"(\[[^\]]*\])", input_string)
|
||||||
|
|
||||||
formatted_parts = []
|
formatted_parts = []
|
||||||
for part in parts:
|
for part in parts:
|
||||||
if part: # Ignore empty strings
|
if part: # Ignore empty strings
|
||||||
# Remove commas and trim spaces for parts not in brackets
|
# Remove commas and trim spaces for parts not in brackets
|
||||||
if not part.startswith('['):
|
if not part.startswith("["):
|
||||||
part = re.sub(r',\s*', ' ', part).strip()
|
part = re.sub(r",\s*", " ", part).strip()
|
||||||
# Enclose the non-bracket part in brackets
|
# Enclose the non-bracket part in brackets
|
||||||
formatted_parts.append(f"[ {part} ]")
|
formatted_parts.append(f"[ {part} ]")
|
||||||
else:
|
else:
|
||||||
# Directly append the part in brackets
|
# Directly append the part in brackets
|
||||||
formatted_parts.append(part)
|
formatted_parts.append(part)
|
||||||
|
|
||||||
return ' '.join(formatted_parts)
|
return " ".join(formatted_parts)
|
||||||
|
|
||||||
|
|
||||||
def is_brackets_and_text_format(s):
|
def is_brackets_and_text_format(s):
|
||||||
# Regular expression to check the pattern
|
# Regular expression to check the pattern
|
||||||
pattern = r'^(\[[^\]]*\]|\s*[^[\]]+\s*)(,\s*|\s+|$)+$'
|
pattern = r"^(\[[^\]]*\]|\s*[^[\]]+\s*)(,\s*|\s+|$)+$"
|
||||||
return bool(re.match(pattern, s))
|
return bool(re.match(pattern, s))
|
||||||
|
|
||||||
|
|
||||||
def reformat(input_string):
|
def reformat(input_string):
|
||||||
input_string = preliminary_reformat(input_string)
|
input_string = preliminary_reformat(input_string)
|
||||||
if check_format(input_string):
|
if check_format(input_string):
|
||||||
return input_string
|
return input_string
|
||||||
# Check if the string is of the first type (numbered list)
|
|
||||||
if is_numbered_line_format(input_string):
|
format_checks = [
|
||||||
print("formated by reformat_list", input_string)
|
(is_numbered_line_format, reformat_list),
|
||||||
return reformat_list(input_string)
|
(is_brackets_and_text_format, reformat_brackets_and_text),
|
||||||
elif is_brackets_and_text_format(input_string):
|
(lambda s: "[" in s and "]" in s, reformat_square_not_full),
|
||||||
print("is_brackets_and_text_format")
|
(lambda s: "/" in s, reformat_slash),
|
||||||
return reformat_brackets_and_text(input_string)
|
(is_pipe_format, reformat_pipe),
|
||||||
# Check if the string is of the second type (already contains square brackets)
|
(is_ists_format, reformat_ists),
|
||||||
elif '[' in input_string and ']' in input_string:
|
(is_section_format, reformat_with_sections),
|
||||||
print("formated by reformat_square_not_full")
|
(is_chunk_format, reformat_chunks),
|
||||||
return reformat_square_not_full(input_string)
|
(is_ists_marker_format, reformat_ists_markers),
|
||||||
# Check for the new type with slashes
|
(is_dash_format, reformat_with_dashes),
|
||||||
elif '/' in input_string:
|
]
|
||||||
print("formated by reformat_slash")
|
|
||||||
return reformat_slash(input_string)
|
for check_func, format_func in format_checks:
|
||||||
elif is_pipe_format(input_string):
|
result = check_and_reformat(input_string, check_func, format_func)
|
||||||
print("formated by reformat_pipe")
|
if result is not None:
|
||||||
return reformat_pipe(input_string)
|
return result
|
||||||
elif is_ists_format(input_string):
|
|
||||||
print("formated by reformat_ists")
|
print("ERROR! reformat did not recognize the format of string! ", input_string)
|
||||||
return reformat_ists(input_string)
|
return False
|
||||||
elif is_section_format(input_string):
|
|
||||||
print("formated by is_section_format")
|
|
||||||
return reformat_with_sections(input_string)
|
|
||||||
elif is_chunk_format(input_string):
|
|
||||||
print("formated by reformat_chunks")
|
|
||||||
return reformat_chunks(input_string)
|
|
||||||
elif is_ists_marker_format(input_string):
|
|
||||||
print("formated by reformat ists markers")
|
|
||||||
return reformat_ists_markers(input_string)
|
|
||||||
# Return the original string if it does not match either type
|
|
||||||
elif is_dash_format(input_string):
|
|
||||||
print("formated by reformat with dashes")
|
|
||||||
return reformat_with_dashes(input_string)
|
|
||||||
print("ERROR! reformat did not recognize the format of string! ", input_string)
|
|
||||||
return False
|
|
||||||
|
|
||||||
def remove_empty_brackets(input_string):
|
def remove_empty_brackets(input_string):
|
||||||
# Regular expression to match empty brackets (with any number of spaces)
|
# Regular expression to match empty brackets (with any number of spaces)
|
||||||
pattern = r'\[\s*\]'
|
pattern = r"\[\s*\]"
|
||||||
|
|
||||||
# Replace empty brackets with an empty string
|
# Replace empty brackets with an empty string
|
||||||
cleaned_string = re.sub(pattern, '', input_string)
|
cleaned_string = re.sub(pattern, "", input_string)
|
||||||
|
|
||||||
# Remove any extra spaces that might be left after removing brackets
|
# Remove any extra spaces that might be left after removing brackets
|
||||||
cleaned_string = re.sub(r'\s{2,}', ' ', cleaned_string).strip()
|
cleaned_string = re.sub(r"\s{2,}", " ", cleaned_string).strip()
|
||||||
|
|
||||||
return cleaned_string
|
return cleaned_string
|
||||||
|
|
||||||
|
|
||||||
def reformat_brackets(input_string):
|
def reformat_brackets(input_string):
|
||||||
formatted_string = re.sub(r'\[\s*(.*?)\s*\]', r'[ \1 ]', input_string)
|
formatted_string = re.sub(r"\[\s*(.*?)\s*\]", r"[ \1 ]", input_string)
|
||||||
return formatted_string
|
return formatted_string
|
||||||
|
|
||||||
def process_sentence(sentence, chunked_sentences):
|
|
||||||
# Define the instruction with line breaks to ensure each line is within 80 characters
|
|
||||||
instruction = (
|
|
||||||
"Please divide the following sentence into iSTS chunks. Try to return chunks as a string in format [chunk 1] [chunk 2] [chunk 3] and so on... Here is the sentence: " + sentence
|
|
||||||
)
|
|
||||||
|
|
||||||
# API call with the instruction variable
|
def generate_instruction(sentence):
|
||||||
response = client.chat.completions.create(
|
return (
|
||||||
messages=[{
|
"Please divide the following sentence into iSTS chunks. Try to return chunks as a string in format "
|
||||||
"role": "user",
|
"[chunk 1] [chunk 2] [chunk 3] and so on... Here is the sentence: " + sentence
|
||||||
"content": instruction,
|
)
|
||||||
}],
|
|
||||||
model="gpt-3.5-turbo",
|
|
||||||
)
|
def call_api(client, instruction):
|
||||||
|
return client.chat.completions.create(
|
||||||
|
messages=[
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": instruction,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
model="gpt-3.5-turbo",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def process_response(response):
|
||||||
|
chunked_sentence = response.choices[0].message.content.strip()
|
||||||
|
return chunked_sentence
|
||||||
|
|
||||||
|
|
||||||
|
def check_and_format_sentence(chunked_sentence):
|
||||||
|
reformated = reformat(chunked_sentence)
|
||||||
|
if reformated == False:
|
||||||
|
print("ERROR! failed to reformat! ", chunked_sentence, reformated)
|
||||||
|
return False
|
||||||
|
reformated = reformated.strip(",")
|
||||||
|
if not is_correct_format(reformated):
|
||||||
|
print("ERROR! wrong format ", chunked_sentence, reformated)
|
||||||
|
return False
|
||||||
|
return reformated
|
||||||
|
|
||||||
|
|
||||||
|
def finalize_formatting(reformated):
|
||||||
|
reformated = remove_empty_brackets(reformated)
|
||||||
|
reformated = reformat_brackets(reformated)
|
||||||
|
return reformated
|
||||||
|
|
||||||
|
|
||||||
|
def process_sentence(sentence, chunked_sentences, client):
|
||||||
|
instruction = generate_instruction(sentence)
|
||||||
|
response = call_api(client, instruction)
|
||||||
|
chunked_sentence = process_response(response)
|
||||||
|
reformated = check_and_format_sentence(chunked_sentence)
|
||||||
|
if not reformated:
|
||||||
|
return False
|
||||||
|
reformated = finalize_formatting(reformated)
|
||||||
|
chunked_sentences.append(reformated)
|
||||||
|
return chunked_sentences
|
||||||
|
|
||||||
chunked_sentence = response.choices[0].message.content
|
|
||||||
chunked_sentence = chunked_sentence.strip()
|
|
||||||
reformated = reformat(chunked_sentence)
|
|
||||||
if reformated == False:
|
|
||||||
print("ERROR! failed to reformat! ", chunked_sentence, reformated)
|
|
||||||
return False
|
|
||||||
reformated = reformated.strip(',')
|
|
||||||
if not is_correct_format(reformated):
|
|
||||||
print("ERROR! wrong format ", chunked_sentence, reformated)
|
|
||||||
return False
|
|
||||||
print("after reformation: ", reformated)
|
|
||||||
reformated = remove_empty_brackets(reformated)
|
|
||||||
reformated = reformat_brackets(reformated)
|
|
||||||
chunked_sentences.append(reformated)
|
|
||||||
return chunked_sentences
|
|
||||||
|
|
||||||
def chunk_sentences(file_path, output_path):
|
def chunk_sentences(file_path, output_path):
|
||||||
# Read the sentences from the file
|
# Read the sentences from the file
|
||||||
with open(file_path, 'r') as file:
|
with open(file_path, "r") as file:
|
||||||
sentences = file.readlines()
|
sentences = file.readlines()
|
||||||
|
|
||||||
# Process each sentence
|
# Process each sentence
|
||||||
chunk_sentences = sentence_loop(sentences)
|
chunk_sentences = sentence_loop(sentences)
|
||||||
print(chunk_sentences)
|
print(chunk_sentences)
|
||||||
# Write the chunked sentences to a new file
|
# Write the chunked sentences to a new file
|
||||||
with open(output_path, 'w') as output_file:
|
with open(output_path, "w") as output_file:
|
||||||
for sentence in chunk_sentences:
|
for sentence in chunk_sentences:
|
||||||
output_file.write(sentence + '\n')
|
output_file.write(sentence + "\n")
|
||||||
|
|
||||||
|
|
||||||
# Usage
|
# Usage
|
||||||
file_path = 'test_goldStandard/student/STSint.testinput.answers-students.sent1.txt'
|
file_path = "test_goldStandard/student/STSint.testinput.answers-students.sent1.txt"
|
||||||
output_path = 'output.txt'
|
output_path = "output.txt"
|
||||||
# Change me to os.environ['API_KEY']
|
# Change me to os.environ['API_KEY']
|
||||||
client = OpenAI(api_key='REDACTED_OPENAI_API_KEY')
|
client = OpenAI(api_key="REDACTED_OPENAI_API_KEY")
|
||||||
|
|
||||||
chunk_sentences(file_path, output_path)
|
chunk_sentences(file_path, output_path)
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user