[MATLAB] Simple TF-IDF implementation

Term-Frequency word weighting scheme is one of most used in normalization of document-term matrices in text mining and information retrieval.

See wikipedia for details.

 

 

function Y = tfidf( X )
% FUNCTION computes TF-IDF weighted word histograms.
%
%   Y = tfidf( X );
%
% INPUT :
%   X        - document-term matrix (documents in columns)
%
% OUTPUT :
%   Y        - TF-IDF weighted document-term matrix
%

% get term frequencies
X = tf(X);

% get inverse document frequencies
I = idf(X);

% apply weights for each document
for j=1:size(X, 2)
    X(:, j) = X(:, j)*I(j);
end

Y = X;


function X = tf(X)
% SUBFUNCTION computes word frequencies

% for every word
for i=1:size(X, 1)

    % get word i counts for all documents
    x = X(i, :);

    % sum all word i occurences in the whole collection
    sumX = sum( x );

    % compute frequency of the word i in the whole collection
    if sumX ~= 0
        X(i, :) = x / sum(x);
    else
        % avoiding NaNs : set zero to never appearing words
        X(i, :) = 0;
    end

end


function I = idf(X)
% SUBFUNCTION computes inverse document frequencies

% m - number of terms or words
% n - number of documents
[m, n]=size(X);

% allocate space for document idf's
I = zeros(n, 1);

% for every document
for j=1:n

    % count non-zero frequency words
    nz = nnz( X(:, j) );

    % if not zero, assign a weight:
    if nz
        I(j) = log( m / nz );
    end

end