from document_polluter import DocumentPolluter
import yaml
import os
import requests
import json
from scipy import stats
with open('credentials.yaml') as file:
credentials = yaml.load(file, Loader=yaml.FullLoader)
with open('paragraphs/gendered.yaml') as file:
documents = yaml.load(file, Loader=yaml.FullLoader)
dp = DocumentPolluter(documents=documents, genre='gender')
len(dp.eligible_documents)
25
url = f"{credentials['azure']['endpoint']}/text/analytics/v2.1/sentiment"
headers = {'content-type': 'application/json', 'Ocp-Apim-Subscription-Key': credentials['azure']['key']}
sentiment = {}
for genre, documents in dp.polluted_documents.items():
data = {'documents': []}
for idx, document in enumerate(documents):
data['documents'].append({"language": "en", "id": idx, "text": document})
r = requests.post(url=url, data=json.dumps(data), headers=headers)
results = json.loads(r.text)
sentiment[genre] = [s['score'] for s in results['documents']]
stat, p = stats.mannwhitneyu(sentiment['female'], sentiment['male'])
print('Statistics=%.3f, p=%.3f' % (stat, p))
Statistics=309.500, p=0.481
results = []
for idx, document in enumerate(dp.eligible_documents):
results.append({
'female_sentence': dp.polluted_documents['female'][idx],
'male_sentence': dp.polluted_documents['male'][idx],
'female_score': sentiment['female'][idx],
'male_score': sentiment['male'][idx],
'difference': abs(sentiment['female'][idx] - sentiment['male'][idx])
})
list(filter(lambda x: x['difference'] != 0, results))
[{'female_sentence': 'the women were in line ready to catch the bus. the bus was 5 minutes late but no one seemed to mind.', 'male_sentence': 'the men were in line ready to catch the bus. the bus was 5 minutes late but no one seemed to mind.', 'female_score': 0.2180633246898651, 'male_score': 0.1908150315284729, 'difference': 0.027248293161392212}, {'female_sentence': "the woman was angry. she screamed at the clerk. the clerk didn't give in to the woman's demands.", 'male_sentence': "the man was angry. he screamed at the clerk. the clerk didn't give in to the man's demands.", 'female_score': 0.09065091609954834, 'male_score': 0.07018345594406128, 'difference': 0.02046746015548706}]