%%%%% Final make good market data clear clc %% import Data & split them % You can use part 1 to transform raw data (.txt form to .mat data) or use part 2 which is .mat data % % 1. transform raw data .txt data to .mat form % % % give data pathes % rowData_Path = 'E:\Data\DataBourse'; % % import texet data to matlab % [StocksDataBase, HaghTaghadom, OtherDataBase,MarketData] = Go_Grab_Datafrom(Data_Path); % 2. load transformed data which are in .mat format load('__DataBase_TSE.mat') %% Prepoceess Data % remove not changed stocks RemoveIdx = zeros(1,size(StocksDataBase,1)); for ii=1:size(StocksDataBase,1) if all((StocksDataBase(ii).AllData.CLOSE) ==1000) || all((StocksDataBase(ii).AllData.LOW) ==0) %%% no Data RemoveIdx(ii) = 1; end end StocksDataBase(find(RemoveIdx),:) = []; %% filter stock wich has low data nWantedDays = 1013; jj=1; for ii = 1:size(StocksDataBase,1) if size( StocksDataBase(ii).AllData ,1)> nWantedDays && StocksDataBase(ii).AllData.DTYYYYMMDD(end) ==20190928 if StocksDataBase(ii).AllData.DTYYYYMMDD(end-nWantedDays+1) ~= StocksDataBase(1).AllData.DTYYYYMMDD(end-nWantedDays+1) continue end HRMarketData(jj).Name = StocksDataBase(ii).Name; HRMarketData(1).DTYYYYMMDD(:,jj) = StocksDataBase(ii).AllData.DTYYYYMMDD(end-nWantedDays+1:end) ; HRMarketData(1).Close(:,jj) = StocksDataBase(ii).AllData.CLOSE(end-nWantedDays+1:end); HRMarketData(1).Vol(:,jj) = StocksDataBase(ii).AllData.VOL(end-nWantedDays+1:end); HRMarketData(1).nTrade(:,jj) = StocksDataBase(ii).AllData.OPENINT1(end-nWantedDays+1:end); jj= jj+1; end end %% remove not changed products for n days D_base = HRMarketData(1).Close; StopDays = zeros(size(D_base)); for jj=1:size(D_base,2) for ii=2:size(D_base,1) if D_base(ii,jj) == D_base(ii-1,jj) StopDays(ii,jj) = StopDays(ii-1,jj)+1; else StopDays(ii,jj) = 0; end end end maxStop = max(StopDays); rmvIdx = find(maxStop>100) ; HRMarketData(rmvIdx) = []; HRMarketData(1).Close(:,rmvIdx) = []; HRMarketData(1).Vol(:,rmvIdx) = []; HRMarketData(1).nTrade(:,rmvIdx) = []; HRMarketData(1).DTYYYYMMDD(:,rmvIdx) = []; %% saving save('A_DataBase_TSE_Filtered', 'HRMarketData')