ENH/DOC fix poly features complexity

larsmans · larsmans · commit f4251347afde · 2014-06-05T17:45:32.000+02:00
Fixes scikit-learn#3191, scikit-learn#3194.
diff --git a/sklearn/preprocessing/data.py b/sklearn/preprocessing/data.py
@@ -4,9 +4,8 @@
 #          Andreas Mueller <amueller@ais.uni-bonn.de>
 # License: BSD 3 clause
 
+from itertools import chain
 import numbers
-import warnings
-import itertools
 
 import numpy as np
 from scipy import sparse
@@ -20,13 +19,15 @@
 from ..utils import safe_asarray
 from ..utils import warn_if_not_float
 from ..utils.extmath import row_norms
+from ..utils.fixes import combinations_with_replacement as comb_w_r
 from ..utils.sparsefuncs_fast import inplace_csr_row_normalize_l1
 from ..utils.sparsefuncs_fast import inplace_csr_row_normalize_l2
 from ..utils.sparsefuncs import inplace_column_scale
 from ..utils.sparsefuncs import mean_variance_axis0
 
 zip = six.moves.zip
 map = six.moves.map
+range = six.moves.range
 
 __all__ = [
     'Binarizer',
@@ -427,8 +428,8 @@ class PolynomialFeatures(BaseEstimator, TransformerMixin):
     Notes
     -----
     Be aware that the number of features in the output array scales
-    exponentially in the number of features of the input array, so this
-    is not suitable for higher-dimensional data.
+    polynomially in the number of features of the input array, and
+    exponentially in the degree. High degrees can cause overfitting.
 
     See :ref:`examples/plot_polynomial_regression.py
     <example_plot_polynomial_regression.py>`
@@ -440,15 +441,11 @@ def __init__(self, degree=2, include_bias=True):
     @staticmethod
     def _power_matrix(n_features, degree, include_bias):
         """Compute the matrix of polynomial powers"""
-        # Find permutations/combinations which add to degree or less
-        deg_min = 0 if include_bias else 1
-        powers = itertools.product(*(range(degree + 1)
-                                     for i in range(n_features)))
-        powers = np.array([c for c in powers if deg_min <= sum(c) <= degree])
-
-        # sort so that the order of the powers makes sense
-        i = np.lexsort(np.vstack([powers.T, powers.sum(1)]))
-        return powers[i]
+        start = int(not include_bias)
+        combn = chain.from_iterable(comb_w_r(range(n_features), i)
+                                    for i in range(start, degree + 1))
+        powers = np.vstack(np.bincount(c, minlength=n_features) for c in combn)
+        return powers
 
     def fit(self, X, y=None):
         """
diff --git a/sklearn/utils/fixes.py b/sklearn/utils/fixes.py
@@ -172,3 +172,27 @@ def sparse_min_max(X, axis):
     # numpy.argpartition was introduced in v 1.8.0
     def argpartition(a, kth, axis=-1, kind='introselect', order=None):
         return np.argsort(a, axis=axis, order=order)
+
+
+try:
+    from itertools import combinations_with_replacement
+except ImportError:
+    # Backport of itertools.combinations_with_replacement for Python 2.6,
+    # from Python 3.4 documentation (http://tinyurl.com/comb-w-r), copyright
+    # Python Software Foundation (https://docs.python.org/3/license.html)
+    def combinations_with_replacement(iterable, r):
+        # combinations_with_replacement('ABC', 2) --> AA AB AC BB BC CC
+        pool = tuple(iterable)
+        n = len(pool)
+        if not n and r:
+            return
+        indices = [0] * r
+        yield tuple(pool[i] for i in indices)
+        while True:
+            for i in reversed(range(r)):
+                if indices[i] != n - 1:
+                    break
+            else:
+                return
+            indices[i:] = [indices[i] + 1] * (r - i)
+            yield tuple(pool[i] for i in indices)
