UgJdZddlZddlZddlmZddlmZddlmZddl Z ddl Z ddl mZddlmZdd lmZmZdd lmZdd lmZd ed edefdZd ed ede jfdZdZ ddZ ddZ ddZdS)z9Implementation of ARFF parsers: via LIAC-ARFF and pandas.N) OrderedDict) Generator)List)_arff)ArffSparseDataType)chunk_generatorget_chunk_n_rows)check_pandas_support) pd_fillna arff_datainclude_columnsreturnctttf}dt|D}t|d|d|dD]a\}}}||vrW|d||d||d||b|S)aObtains several columns from sparse ARFF representation. Additionally, the column indices are re-labelled, given the columns that are not included. (e.g., when including [1, 2, 3], the columns will be relabelled to [0, 1, 2]). Parameters ---------- arff_data : tuple A tuple of three lists of equal size; first list indicating the value, second the x coordinate and the third the y coordinate. include_columns : list A list of columns to include. Returns ------- arff_data_new : tuple Subset of arff data with only the include columns indicated by the include_columns argument. ci|]\}}|| Sr.0 array_idx column_idxs \/var/www/surfInsights/venv3-11/lib/python3.11/site-packages/sklearn/datasets/_arff_parser.py z)_split_sparse_columns..++"7)Z Irr)list enumeratezipappend)r r arff_data_newreindexed_columnsvalrow_idxcol_idxs r_split_sparse_columnsr%s.*.(@M;D_;U;U"%Yq\9Q<1!N!N@@Wg o % % !  # #C ( ( ( !  # #G , , , !  # #$5g$> ? ? ? rc@t|ddz}|t|f}dt|D}tj|tj}t |d|d|dD]\}}}||vr |||||f<|S)Nrci|]\}}|| Srrrs rrz)_sparse_data_to_array..=rrdtyperr)maxlenrnpemptyfloat64r) r rnum_obsy_shaper!yr"r#r$s r_sparse_data_to_arrayr26s )A,!#GO,,-G;D_;U;U  +++A!$Yq\9Q<1!N!N99Wg o % %58Ag(11 2 Hrc||}t|dkr ||}n$t|dkr||d}nd}||fS)a Post process a dataframe to select the desired columns in `X` and `y`. Parameters ---------- frame : dataframe The dataframe to split into `X` and `y`. feature_names : list of str The list of feature names to populate `X`. target_names : list of str The list of target names to populate `y`. Returns ------- X : dataframe The dataframe containing the features. y : {series, dataframe} or None The series or dataframe containing the target. rrrN)r+)frame feature_names target_namesXr1s r_post_process_framer8Hs^, mA <A ,  \  a   ,q/ "  a4Krc| "#$d}||}|dkr tjn tj}|dk } tj||| } ||z##fd| dD"|dkrt d} t | d} t | } t| d}| |g| d }| d  }t|}#fd | D}||g}t| d|D]3}|| || d |4t|dkr)|d|dj|d<| |d }t'| |}~~i}|jD]\}|d}|dkrd||<.|dkrd||<L|j|||<]||}t-|||\}$n<| d}fd|D}fd|D}t/|t0r|t3d|ddkrd}n|d|dz}t5jt8j|d|}|j|}|dd|f}|dd|f$nt/|t@rtC||}tE|ddz}|t|f} tFj$%|d|d|dff| t4j&}|'}tQ||$nt3dtS|"fd |D}!|!sn\tU|!r/t5j+"$fd!tY|D$nt[|!rt3d"$j.ddkr$d#$n$j.ddkrd$|dkr|$|dfS|$d"fS)$aARFF parser using the LIAC-ARFF library coded purely in Python. This parser is quite slow but consumes a generator. Currently it is needed to parse sparse datasets. For dense datasets, it is recommended to instead use the pandas-based parser, although it does not always handles the dtypes exactly the same. Parameters ---------- gzip_file : GzipFile instance The file compressed to be read. output_arrays_type : {"numpy", "sparse", "pandas"} The type of the arrays that will be returned. The possibilities ara: - `"numpy"`: both `X` and `y` will be NumPy arrays; - `"sparse"`: `X` will be sparse matrix and `y` will be a NumPy array; - `"pandas"`: `X` will be a pandas DataFrame and `y` will be either a pandas Series or DataFrame. columns_info : dict The information provided by OpenML regarding the columns of the ARFF file. feature_names_to_select : list of str A list of the feature names to be selected. target_names_to_select : list of str A list of the target names to be selected. Returns ------- X : {ndarray, sparse matrix, dataframe} The data matrix. y : {ndarray, dataframe, series} The target. frame : dataframe or None A dataframe containing both `X` and `y`. `None` if `output_array_type != "pandas"`. categories : list of str or None The names of the features that are categorical. `None` if `output_array_type == "pandas"`. c3BK|D]}|dVdS)Nutf-8)decode) gzip_filelines r_io_to_generatorz+_liac_arff_parser.._io_to_generators: ' 'D++g&& & & & & ' 'rsparsepandas) return_typeencode_nominalcNi|]!\}}t|tr|v||"Sr) isinstancer)rnamecatcolumns_to_selects rrz%_liac_arff_parser..sI D# c4 &*->%>%> c%>%>%>r attributeszfetch_openml with as_frame=TruedataF)columnscopyT)deepcg|]}|v| SrrrcolrHs r z%_liac_arff_parser..s$TTT33BS;S;S3;S;S;Srrrr) ignore_index data_typeintegerInt64nominalcategorycFg|]}t|dSindexintrcol_nameopenml_columns_infos rrQz%_liac_arff_parser..s<% % %  #H-g6 7 7% % % rcFg|]}t|dSrYr[r]s rrQz%_liac_arff_parser..s<$ $ $  #H-g6 7 7$ $ $ rNz6shape must be provided when arr['data'] is a Generatorr.)r)count)shaper)z-Unexpected type for data obtained from arff: ch|]}|vSrr)rr^ categoriess r z$_liac_arff_parser..s+   '/H "   rc g|]j\}}tjtj|ddd||dzft dkS)Or(NrF)rL)r,takeasarraypopastyper\)rir^rer1s rrQz%_liac_arff_parser..s $8 G :>>(#;#;3GGG!!!QQY,..s.??rzAMix of nominal and non-nominal targets is not currently supported)ra)/rCOO DENSE_GENloadr rrkeysnext DataFrame memory_usagesumr r rr+rldtypesconcatr rKlowerr8rEr ValueErrorr,fromiter itertoolschain from_iterablereshapetupler%r*spr@ coo_matrixr.tocsrr2typeallhstackranyrc)%r=output_arrays_typer_feature_names_to_selecttarget_names_to_selectrcr?streamrBrCarff_containerpd columns_info columns_names first_rowfirst_df row_bytes chunksizecolumns_to_keepdfsrJr4rvrF column_dtyper7r feature_indices_to_selecttarget_indices_to_selectrb arff_data_Xr/X_shapeis_classificationrerHr1s% ` @@@r_liac_arff_parserrhsn''' i ( (F 2X==%))5?K-89NZKN02HH' 5J X%% !"C D D">,#?@@ \..0011 /00 << ]<OO))t)4488:: $Y// UTTT-TTT()#N6$:IFF  D JJ T=u EEoV     s88q==V]]3q6=11CF  #D 11"e$$ M 2 2D.t4[AL!!##y00 't ##%%22)t $|D1t  V$$" *,B  11#6* % % % % 3% % % !$ $ $ $ 2$ $ $  i + + } LQx2~~a58+;--i88D  4<'DQQQ112AQQQ001AA  5 ) ) / ;TUUK)A,''!+G$= > >?G $$Q+a.+a.!ABj%A  A%i1IJJAAQYQQ     3I   !   " # #   (11G'H'H AA" # # S  71:?? %  AA WQZ1__AX%%!UD  az !!rc "ddl|D]>}|ddrn?i|D]K}||d}|dkrd|<.|dkrd |<Lfd t |D} dd d gd d ddd| d } i| |pi}j|fi|} d|D| _n-#t$r } j d| d} ~ wwxYw||zfd| jD} | | } tj dfd}fd| j D}|D]%}| |j|| |<&t!| ||\}}|dkr||| dfS||}}fd| j D}||d|fS)a^ARFF parser using `pandas.read_csv`. This parser uses the metadata fetched directly from OpenML and skips the metadata headers of ARFF file itself. The data is loaded as a CSV file. Parameters ---------- gzip_file : GzipFile instance The GZip compressed file with the ARFF formatted payload. output_arrays_type : {"numpy", "sparse", "pandas"} The type of the arrays that will be returned. The possibilities are: - `"numpy"`: both `X` and `y` will be NumPy arrays; - `"sparse"`: `X` will be sparse matrix and `y` will be a NumPy array; - `"pandas"`: `X` will be a pandas DataFrame and `y` will be either a pandas Series or DataFrame. openml_columns_info : dict The information provided by OpenML regarding the columns of the ARFF file. feature_names_to_select : list of str A list of the feature names to be selected to build `X`. target_names_to_select : list of str A list of the target names to be selected to build `y`. read_csv_kwargs : dict, default=None Keyword arguments to pass to `pandas.read_csv`. It allows to overwrite the default options. Returns ------- X : {ndarray, sparse matrix, dataframe} The data matrix. y : {ndarray, dataframe, series} The target. frame : dataframe or None A dataframe containing both `X` and `y`. `None` if `output_array_type != "pandas"`. categories : list of str or None The names of the features that are categorical. `None` if `output_array_type == "pandas"`. rNr;z@datarSrTrUrVrWc0i|]\}}|v ||Srr)rr$rFrvs rrz'_pandas_arff_parser..~s4 GT 6>>  >>rF?%"T\) header index_col na_valueskeep_default_nacomment quotecharskipinitialspace escapecharr)cg|]}|Srr)rrFs rrQz'_pandas_arff_parser..s>>>$>>>rzwThe number of columns provided by OpenML does not match the number of columns inferred by pandas when reading the file.cg|]}|v| SrrrOs rrQz'_pandas_arff_parser..s$PPPss>O7O7Os7O7O7Orz^'(?P.*)'$c`tj|}||S|dS)Ncontents)researchgroup) input_stringmatchsingle_quote_patterns rstrip_single_quotesz0_pandas_arff_parser..strip_single_quotess2 . == = {{:&&&rcDg|]\}}t|j|Sr)rECategoricalDtyperrFr)rs rrQz'_pandas_arff_parser..s@ D% eR0 1 1 rrActi|]4\}}t|j||j5Sr)rErretolistrs rrz'_pandas_arff_parser..sQ D% eR0 1 1 e%%''r)rAr<rx startswithrread_csvrKryerrors ParserErrorrcompilervitemsrGrename_categoriesr8to_numpy)r=rr_rrread_csv_kwargsr>rFrdtypes_positionaldefault_read_csv_kwargsr4excrrcategorical_columnsrPr7r1rerHrvrrs @@@@r_pandas_arff_parserr4sp ;;w   % % ' ' 2 27 ; ;  E F#&&*40=     9 , ,#F4LL    ! !Y . .%F4L&':;;U  "  M0L_5JLO BK 5 5_ 5 5E  ?>*=>>> i## @    02HHPPPPemPPPO / "E:&=>>''''' <--// #KK3Z^556IJJc u&=?U V VDAqX%%!UD  zz||QZZ\\1 <--//J az !!sC++ D5DDc|dkrt||||||S|dkrt||||||Std|d)a6Load a compressed ARFF file using a given parser. Parameters ---------- gzip_file : GzipFile instance The file compressed to be read. parser : {"pandas", "liac-arff"} The parser used to parse the ARFF file. "pandas" is recommended but only supports loading dense datasets. output_type : {"numpy", "sparse", "pandas"} The type of the arrays that will be returned. The possibilities ara: - `"numpy"`: both `X` and `y` will be NumPy arrays; - `"sparse"`: `X` will be sparse matrix and `y` will be a NumPy array; - `"pandas"`: `X` will be a pandas DataFrame and `y` will be either a pandas Series or DataFrame. openml_columns_info : dict The information provided by OpenML regarding the columns of the ARFF file. feature_names_to_select : list of str A list of the feature names to be selected. target_names_to_select : list of str A list of the target names to be selected. read_csv_kwargs : dict, default=None Keyword arguments to pass to `pandas.read_csv`. It allows to overwrite the default options. Returns ------- X : {ndarray, sparse matrix, dataframe} The data matrix. y : {ndarray, dataframe, series} The target. frame : dataframe or None A dataframe containing both `X` and `y`. `None` if `output_array_type != "pandas"`. categories : list of str or None The names of the features that are categorical. `None` if `output_array_type == "pandas"`. z liac-arffrAzUnknown parser: 'z%'. Should be 'liac-arff' or 'pandas'.)rrry)r=parser output_typer_rrrcrs rload_arff_from_gzip_filersv    # "      8  "    # "      M M M M   r)N)NN)__doc__r{r collectionsrcollections.abcrtypingrnumpyr,scipyr externalsrexternals._arffrutils._chunkingr r utils._optional_dependenciesr utils.fixesr r%ndarrayr2r8rrrrrrrs?? ######%%%%%%000000??????????????###### ! 48     F ! 48 Z    $L  I"I"I"I"d U"U"U"U"~ P P P P P P r