Path: blob/main/a2/utils/__pycache__/treebank.cpython-39.pyc
1003 views
a
��cx � @ s2 d dl Z d dlZd dlZd dlZG dd� d�ZdS )� Nc @ s� e Zd Zd'dd�Zdd� Zdd� Zd d
� Zd d � Zd(dd�Zdd� Z dd� Z
dd� Z dd� Z dd� Z
dd� Zd d � Zd)d d �Zd!d"� Zd#d$� Zd%d&� ZdS )*�StanfordSentimentN�@B c C s |sd}|| _ || _d S )Nz(utils/datasets/stanfordSentimentTreebank)�path� tablesize)�selfr r � r �@/Users/yimingwang/Desktop/cs224n/assignment/a2/utils/treebank.py�__init__
s zStanfordSentiment.__init__c C s� t | d�r| jr| jS t� }t� }d}g }d}| �� D ]R}|D ]H}|d7 }||vrv|||< ||g7 }d||< |d7 }q>|| d7 < q>q6||d< |dg7 }d|d< |d7 }|| _|| _|| _|| _| jS )N�_tokensr � ZUNK)�hasattrr
�dict� sentences�
_tokenfreq�
_wordcount�
_revtokens)r �tokensZ tokenfreqZ wordcountZ revtokens�idx�sentence�wr r r r s2
zStanfordSentiment.tokensc C s� t | d�r| jr| jS g }t| jd d��P}d}|D ]6}|rBd}q4|�� �� dd � }|dd� |D �g7 }q4W d � n1 s�0 Y || _t�d d� |D ��| _t� | j�| _
| jS )
N�
_sentencesz/datasetSentences.txt�rTFr c S s g | ] }|� � �qS r )�lower��.0r r r r �
<listcomp>? � z/StanfordSentiment.sentences.<locals>.<listcomp>c S s g | ] }t |��qS r ��len�r �sr r r r B r ) r r �openr �strip�split�np�arrayZ _sentlengths�cumsumZ _cumsentlen)r r �f�first�line�splittedr r r r 1 s 4zStanfordSentiment.sentencesc C s. t | d�r| jr| jS t| �� �| _| jS d S )N�
_numSentences)r r+ r r �r r r r � numSentencesG s z StanfordSentiment.numSentencesc s` t | d�r| jr| jS | �� }| �� � | �� �� �fdd�|d D �}dd� |D �}|| _| jS )N�
_allsentencesc s g | ]}� �fd d�|D ��qS )c s4 g | ],}d � �| ks,t � � � �| kr|�qS )r )�randomr ��
rejectProbr r r r U s $�z=StanfordSentiment.allSentences.<locals>.<listcomp>.<listcomp>r r r0 r r r U s �z2StanfordSentiment.allSentences.<locals>.<listcomp>� c S s g | ]}t |�d kr|�qS )r r r r r r r Y r )r r. r r1 r )r r Z allsentencesr r0 r � allSentencesN s �z StanfordSentiment.allSentences� c s� | � � }t�dt|�d �}|| }t�dt|�d �}|td|| �|� }|d t|�k r�|||d tt|�|| d �� 7 }|| � � fdd�|D �}t|�dkr�� |fS | �|�S d S )Nr r c s g | ]}|� kr|�qS r r r �Z
centerwordr r r j r z6StanfordSentiment.getRandomContext.<locals>.<listcomp>)r3 r/ �randintr �max�min�getRandomContext)r �CZallsentZsentIDZsentZwordID�contextr r5 r r9 _ s & z"StanfordSentiment.getRandomContextc
C s~ t | d�r| jr| jS t� }d}t| jd d��R}|D ]<}|�� }|sHq6|�d�}t|d �||d �� < |d7 }q6W d � n1 s�0 Y dg| }t| jd d��X}d }|D ]>}|r�d
}q�|�� }|s�q�|�d�}t |d �|t|d �< q�W d � n1 �s 0 Y dg| �
� }| � � } t | �
� �D ]8}
| |
} d �
| ��d d
��dd�} ||| ||
< �q8|| _| jS )N� _sent_labelsr z/dictionary.txtr �|r � z/sentiment_labels.txtTF� z-lrb-�(z-rrb-�))r r< r
r! r r" r# �intr �floatr- r �range�join�replace)
r Z
dictionaryZphrasesr' r) r* �labelsr( � sent_labelsr �ir Z full_sentr r r rH q s<
(
:z StanfordSentiment.sent_labelsc C s� t | d�r| jr| jS dd� td�D �}t| jd d��`}d}|D ]F}|rPd}qB|�� �d �}|t|d
�d
t|d �d
g7 < qBW d � n1 s�0 Y || _| jS ) N�_splitc S s g | ]}g �qS r r �r rI r r r r � r z3StanfordSentiment.dataset_split.<locals>.<listcomp>� z/datasetSplit.txtr TF�,r r )r rJ rD r! r r" r# rB )r r# r' r( r) r* r r r �
dataset_split� s Jz StanfordSentiment.dataset_splitc C sF | � � }|d t�dt|d �d � }| �� | | �| �� | �fS �Nr r )rN r/ r6 r r �
categorifyrH )r r# ZsentIdr r r �getRandomTrainSentence� s z(StanfordSentiment.getRandomTrainSentencec C s8 |dkr dS |dkrdS |dkr$dS |dkr0dS d S d S )
Ng�������?r g�������?r g333333�?� g�������?rL � r )r �labelr r r rP � s z StanfordSentiment.categorifyc C s
| � d�S )NrR ��getSplitSentencesr, r r r �getDevSentences� s z!StanfordSentiment.getDevSentencesc C s
| � d�S )Nr rU r, r r r �getTestSentences� s z"StanfordSentiment.getTestSentencesc C s
| � d�S )Nr rU r, r r r �getTrainSentences� s z#StanfordSentiment.getTrainSentencesr c s � � � }� fdd�|| D �S )Nc s* g | ]"}� � � | � �� �� | �f�qS r )r rP rH rK r, r r r � r z7StanfordSentiment.getSplitSentences.<locals>.<listcomp>)rN )r r# Zds_splitr r, r rV � s z#StanfordSentiment.getSplitSentencesc C s� t | d�r| jd ur| jS t| �� �}t�|f�}| �� d}t|�D ]D}| j| }|| j v rvd| j | }|d }nd}|||< |d7 }qF|t�
|� }t� |�| j }dg| j | _d}t| j �D ]$}||| kr�|d7 }q�|| j|< q�| jS )N� _sampleTabler � �?g �?r> r )
r rZ r r r$ �zerosr3 rD r r �sumr& r )r �nTokensZ samplingFreqrI r �freq�jr r r � sampleTable� s.
z StanfordSentiment.sampleTablec C s� t | d�r| jd ur| jS d| j }t| �� �}t�|f�}t|�D ]8}| j| }d| j | }t
ddt� || � �||< qD|| _| jS )N� _rejectProbg�h㈵��>r[ r r ) r rb r r r r$ r\ rD r r r7 �sqrt)r � thresholdr^ r1 rI r r_ r r r r1 � s
z StanfordSentiment.rejectProbc C s | � � t�d| jd � S rO )ra r/ r6 r r, r r r �sampleTokenIdx� s z StanfordSentiment.sampleTokenIdx)Nr )r4 )r )�__name__�
__module__� __qualname__r r r r- r3 r9 rH rN rQ rP rW rX rY rV ra r1 re r r r r r s"
%
r )�pickle�numpyr$ �osr/ r r r r r �<module> s