Skip to content

bpo-4963: Fix for initialization and non-deterministic behavior issues in mimetypes #3062

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Jun 24, 2019
Merged

bpo-4963: Fix for initialization and non-deterministic behavior issues in mimetypes #3062

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
ecabb1c
Addresses mimetypes initialization problems and non-deterministic beh…
davidkhess May 13, 2017
0e8c48f
Fix edge case with init.
davidkhess May 14, 2017
10abe19
Fix another edge case.
davidkhess May 14, 2017
2486b92
Fix whitespace issue with added test.
davidkhess Aug 10, 2017
384178c
Capture the new init() behavior in the documentation.
davidkhess Aug 10, 2017
32cd666
Merge remote-tracking branch 'python/master' into fix-issue-4963
davidkhess Aug 10, 2017
d2097be
Tweak to the documentation per review feedback.
davidkhess Aug 11, 2017
1f45e83
Merge remote-tracking branch 'python/master' into fix-issue-4963
davidkhess Aug 15, 2017
f9e59f9
Add news entry.
davidkhess Aug 15, 2017
59d1128
Remove use of OrderedDict
davidkhess May 21, 2019
799ce54
Merge remote-tracking branch 'python/master' into fix-issue-4963
davidkhess May 21, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Addresses mimetypes initialization problems and non-deterministic beh… 
…avior.
  • Loading branch information
@davidkhess
davidkhess committed May 13, 2017
commit ecabb1cb57e7e066a693653f485f2f687dcc7f6b
312 changes: 158 additions & 154 deletions Lib/mimetypes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
except ImportError:
_winreg = None

from collections import OrderedDict
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This no longer needed in 3.7+.

Copy link
Contributor Author

@davidkhess davidkhess Apr 10, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should I remove all usage of OrderedDict from the PR and let others add it back for back porting it to 2.7? Or should I do something like set OrderedDict = dict if Python version >= 3.7?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The order of the items in the list is a critical part of the fix - I'd vote for retaining the explicit use of OrderedDict.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Adding OrderedDict makes this PR more complex, and the resulting code clumsy, and this is absolutely unneeded in 3.7+. I think it would be better to merge a simple fix into master and 3.7, and prepare more complex backport for 3.6.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since 3.6 is only receiving security fixes, there's no need to worry about the backport to 3.6 anymore. So, you can remove the OrderedDict as this will only be in 3.7 and 3.8.


__all__ = [
"knownfiles", "inited", "MimeTypes",
"guess_type", "guess_all_extensions", "guess_extension",
Expand Down Expand Up @@ -66,13 +68,13 @@ class MimeTypes:
def __init__(self, filenames=(), strict=True):
if not inited:
init()
self.encodings_map = encodings_map.copy()
self.suffix_map = suffix_map.copy()
self.encodings_map = _encodings_map_default.copy()
self.suffix_map = _suffix_map_default.copy()
self.types_map = ({}, {}) # dict for (non-strict, strict)
self.types_map_inv = ({}, {})
for (ext, type) in types_map.items():
for (ext, type) in _types_map_default.items():
self.add_type(type, ext, True)
for (ext, type) in common_types.items():
for (ext, type) in _common_types_default.items():
self.add_type(type, ext, False)
for name in filenames:
self.read(name, strict)
Expand Down Expand Up @@ -345,11 +347,15 @@ def init(files=None):
global suffix_map, types_map, encodings_map, common_types
global inited, _db
inited = True # so that MimeTypes.__init__() doesn't call us again
db = MimeTypes()

if files is None:
db = MimeTypes()
if _winreg:
db.read_windows_registry()
files = knownfiles
else:
db = _db

for file in files:
if os.path.isfile(file):
db.read(file)
Expand All @@ -373,12 +379,12 @@ def read_mime_types(file):


def _default_mime_types():
global suffix_map
global encodings_map
global types_map
global common_types
global suffix_map, _suffix_map_default
global encodings_map, _encodings_map_default
global types_map, _types_map_default
global common_types, _common_types_default

suffix_map = {
suffix_map = _suffix_map_default = {
'.svgz': '.svg.gz',
'.tgz': '.tar.gz',
'.taz': '.tar.gz',
Expand All @@ -387,7 +393,7 @@ def _default_mime_types():
'.txz': '.tar.xz',
}

encodings_map = {
encodings_map = _encodings_map_default = {
'.gz': 'gzip',
'.Z': 'compress',
'.bz2': 'bzip2',
Expand All @@ -398,154 +404,152 @@ def _default_mime_types():
# at http://www.iana.org/assignments/media-types
# or extensions, i.e. using the x- prefix

# If you add to these, please keep them sorted!
types_map = {
'.a' : 'application/octet-stream',
'.ai' : 'application/postscript',
'.aif' : 'audio/x-aiff',
'.aifc' : 'audio/x-aiff',
'.aiff' : 'audio/x-aiff',
'.au' : 'audio/basic',
'.avi' : 'video/x-msvideo',
'.bat' : 'text/plain',
'.bcpio' : 'application/x-bcpio',
'.bin' : 'application/octet-stream',
'.bmp' : 'image/x-ms-bmp',
'.c' : 'text/plain',
# Duplicates :(
'.cdf' : 'application/x-cdf',
'.cdf' : 'application/x-netcdf',
'.cpio' : 'application/x-cpio',
'.csh' : 'application/x-csh',
'.css' : 'text/css',
'.csv' : 'text/csv',
'.dll' : 'application/octet-stream',
'.doc' : 'application/msword',
'.dot' : 'application/msword',
'.dvi' : 'application/x-dvi',
'.eml' : 'message/rfc822',
'.eps' : 'application/postscript',
'.etx' : 'text/x-setext',
'.exe' : 'application/octet-stream',
'.gif' : 'image/gif',
'.gtar' : 'application/x-gtar',
'.h' : 'text/plain',
'.hdf' : 'application/x-hdf',
'.htm' : 'text/html',
'.html' : 'text/html',
'.ico' : 'image/vnd.microsoft.icon',
'.ief' : 'image/ief',
'.jpe' : 'image/jpeg',
'.jpeg' : 'image/jpeg',
'.jpg' : 'image/jpeg',
'.js' : 'application/javascript',
'.ksh' : 'text/plain',
'.latex' : 'application/x-latex',
'.m1v' : 'video/mpeg',
'.m3u' : 'application/vnd.apple.mpegurl',
'.m3u8' : 'application/vnd.apple.mpegurl',
'.man' : 'application/x-troff-man',
'.me' : 'application/x-troff-me',
'.mht' : 'message/rfc822',
'.mhtml' : 'message/rfc822',
'.mif' : 'application/x-mif',
'.mov' : 'video/quicktime',
'.movie' : 'video/x-sgi-movie',
'.mp2' : 'audio/mpeg',
'.mp3' : 'audio/mpeg',
'.mp4' : 'video/mp4',
'.mpa' : 'video/mpeg',
'.mpe' : 'video/mpeg',
'.mpeg' : 'video/mpeg',
'.mpg' : 'video/mpeg',
'.ms' : 'application/x-troff-ms',
'.nc' : 'application/x-netcdf',
'.nws' : 'message/rfc822',
'.o' : 'application/octet-stream',
'.obj' : 'application/octet-stream',
'.oda' : 'application/oda',
'.p12' : 'application/x-pkcs12',
'.p7c' : 'application/pkcs7-mime',
'.pbm' : 'image/x-portable-bitmap',
'.pdf' : 'application/pdf',
'.pfx' : 'application/x-pkcs12',
'.pgm' : 'image/x-portable-graymap',
'.pl' : 'text/plain',
'.png' : 'image/png',
'.pnm' : 'image/x-portable-anymap',
'.pot' : 'application/vnd.ms-powerpoint',
'.ppa' : 'application/vnd.ms-powerpoint',
'.ppm' : 'image/x-portable-pixmap',
'.pps' : 'application/vnd.ms-powerpoint',
'.ppt' : 'application/vnd.ms-powerpoint',
'.ps' : 'application/postscript',
'.pwz' : 'application/vnd.ms-powerpoint',
'.py' : 'text/x-python',
'.pyc' : 'application/x-python-code',
'.pyo' : 'application/x-python-code',
'.qt' : 'video/quicktime',
'.ra' : 'audio/x-pn-realaudio',
'.ram' : 'application/x-pn-realaudio',
'.ras' : 'image/x-cmu-raster',
'.rdf' : 'application/xml',
'.rgb' : 'image/x-rgb',
'.roff' : 'application/x-troff',
'.rtx' : 'text/richtext',
'.sgm' : 'text/x-sgml',
'.sgml' : 'text/x-sgml',
'.sh' : 'application/x-sh',
'.shar' : 'application/x-shar',
'.snd' : 'audio/basic',
'.so' : 'application/octet-stream',
'.src' : 'application/x-wais-source',
'.sv4cpio': 'application/x-sv4cpio',
'.sv4crc' : 'application/x-sv4crc',
'.svg' : 'image/svg+xml',
'.swf' : 'application/x-shockwave-flash',
'.t' : 'application/x-troff',
'.tar' : 'application/x-tar',
'.tcl' : 'application/x-tcl',
'.tex' : 'application/x-tex',
'.texi' : 'application/x-texinfo',
'.texinfo': 'application/x-texinfo',
'.tif' : 'image/tiff',
'.tiff' : 'image/tiff',
'.tr' : 'application/x-troff',
'.tsv' : 'text/tab-separated-values',
'.txt' : 'text/plain',
'.ustar' : 'application/x-ustar',
'.vcf' : 'text/x-vcard',
'.wav' : 'audio/x-wav',
'.webm' : 'video/webm',
'.wiz' : 'application/msword',
'.wsdl' : 'application/xml',
'.xbm' : 'image/x-xbitmap',
'.xlb' : 'application/vnd.ms-excel',
# Duplicates :(
'.xls' : 'application/excel',
'.xls' : 'application/vnd.ms-excel',
'.xml' : 'text/xml',
'.xpdl' : 'application/xml',
'.xpm' : 'image/x-xpixmap',
'.xsl' : 'application/xml',
'.xwd' : 'image/x-xwindowdump',
'.zip' : 'application/zip',
}
# If you add to these, please keep them sorted by mime type.
# Make sure the entry with the preferred file extension for a particular mime type
# appears before any others of the same mimetype.
types_map = _types_map_default = OrderedDict([
('.js' , 'application/javascript'),
('.doc' , 'application/msword'),
('.dot' , 'application/msword'),
('.wiz' , 'application/msword'),
('.bin' , 'application/octet-stream'),
('.a' , 'application/octet-stream'),
('.dll' , 'application/octet-stream'),
('.exe' , 'application/octet-stream'),
('.o' , 'application/octet-stream'),
('.obj' , 'application/octet-stream'),
('.so' , 'application/octet-stream'),
('.oda' , 'application/oda'),
('.pdf' , 'application/pdf'),
('.p7c' , 'application/pkcs7-mime'),
('.ps' , 'application/postscript'),
('.ai' , 'application/postscript'),
('.eps' , 'application/postscript'),
('.m3u' , 'application/vnd.apple.mpegurl'),
('.m3u8' , 'application/vnd.apple.mpegurl'),
('.xls' , 'application/vnd.ms-excel'),
('.xlb' , 'application/vnd.ms-excel'),
('.ppt' , 'application/vnd.ms-powerpoint'),
('.pot' , 'application/vnd.ms-powerpoint'),
('.ppa' , 'application/vnd.ms-powerpoint'),
('.pps' , 'application/vnd.ms-powerpoint'),
('.pwz' , 'application/vnd.ms-powerpoint'),
('.bcpio' , 'application/x-bcpio'),
('.cpio' , 'application/x-cpio'),
('.csh' , 'application/x-csh'),
('.dvi' , 'application/x-dvi'),
('.gtar' , 'application/x-gtar'),
('.hdf' , 'application/x-hdf'),
('.latex' , 'application/x-latex'),
('.mif' , 'application/x-mif'),
('.cdf' , 'application/x-netcdf'),
('.nc' , 'application/x-netcdf'),
('.p12' , 'application/x-pkcs12'),
('.pfx' , 'application/x-pkcs12'),
('.ram' , 'application/x-pn-realaudio'),
('.pyc' , 'application/x-python-code'),
('.pyo' , 'application/x-python-code'),
('.sh' , 'application/x-sh'),
('.shar' , 'application/x-shar'),
('.swf' , 'application/x-shockwave-flash'),
('.sv4cpio', 'application/x-sv4cpio'),
('.sv4crc' , 'application/x-sv4crc'),
('.tar' , 'application/x-tar'),
('.tcl' , 'application/x-tcl'),
('.tex' , 'application/x-tex'),
('.texi' , 'application/x-texinfo'),
('.texinfo', 'application/x-texinfo'),
('.roff' , 'application/x-troff'),
('.t' , 'application/x-troff'),
('.tr' , 'application/x-troff'),
('.man' , 'application/x-troff-man'),
('.me' , 'application/x-troff-me'),
('.ms' , 'application/x-troff-ms'),
('.ustar' , 'application/x-ustar'),
('.src' , 'application/x-wais-source'),
('.xsl' , 'application/xml'),
('.rdf' , 'application/xml'),
('.wsdl' , 'application/xml'),
('.xpdl' , 'application/xml'),
('.zip' , 'application/zip'),
('.au' , 'audio/basic'),
('.snd' , 'audio/basic'),
('.mp3' , 'audio/mpeg'),
('.mp2' , 'audio/mpeg'),
('.aif' , 'audio/x-aiff'),
('.aifc' , 'audio/x-aiff'),
('.aiff' , 'audio/x-aiff'),
('.ra' , 'audio/x-pn-realaudio'),
('.wav' , 'audio/x-wav'),
('.gif' , 'image/gif'),
('.ief' , 'image/ief'),
('.jpg' , 'image/jpeg'),
('.jpe' , 'image/jpeg'),
('.jpeg' , 'image/jpeg'),
('.png' , 'image/png'),
('.svg' , 'image/svg+xml'),
('.tiff' , 'image/tiff'),
('.tif' , 'image/tiff'),
('.ico' , 'image/vnd.microsoft.icon'),
('.ras' , 'image/x-cmu-raster'),
('.bmp' , 'image/x-ms-bmp'),
('.pnm' , 'image/x-portable-anymap'),
('.pbm' , 'image/x-portable-bitmap'),
('.pgm' , 'image/x-portable-graymap'),
('.ppm' , 'image/x-portable-pixmap'),
('.rgb' , 'image/x-rgb'),
('.xbm' , 'image/x-xbitmap'),
('.xpm' , 'image/x-xpixmap'),
('.xwd' , 'image/x-xwindowdump'),
('.eml' , 'message/rfc822'),
('.mht' , 'message/rfc822'),
('.mhtml' , 'message/rfc822'),
('.nws' , 'message/rfc822'),
('.css' , 'text/css'),
('.csv' , 'text/csv'),
('.html' , 'text/html'),
('.htm' , 'text/html'),
('.txt' , 'text/plain'),
('.bat' , 'text/plain'),
('.c' , 'text/plain'),
('.h' , 'text/plain'),
('.ksh' , 'text/plain'),
('.pl' , 'text/plain'),
('.rtx' , 'text/richtext'),
('.tsv' , 'text/tab-separated-values'),
('.py' , 'text/x-python'),
('.etx' , 'text/x-setext'),
('.sgm' , 'text/x-sgml'),
('.sgml' , 'text/x-sgml'),
('.vcf' , 'text/x-vcard'),
('.xml' , 'text/xml'),
('.mp4' , 'video/mp4'),
('.mpeg' , 'video/mpeg'),
('.m1v' , 'video/mpeg'),
('.mpa' , 'video/mpeg'),
('.mpe' , 'video/mpeg'),
('.mpg' , 'video/mpeg'),
('.mov' , 'video/quicktime'),
('.qt' , 'video/quicktime'),
('.webm' , 'video/webm'),
('.avi' , 'video/x-msvideo'),
('.movie' , 'video/x-sgi-movie'),
])

# These are non-standard types, commonly found in the wild. They will
# only match if strict=0 flag is given to the API methods.

# Please sort these too
common_types = {
'.jpg' : 'image/jpg',
'.mid' : 'audio/midi',
'.midi': 'audio/midi',
'.pct' : 'image/pict',
'.pic' : 'image/pict',
'.pict': 'image/pict',
'.rtf' : 'application/rtf',
'.xul' : 'text/xul'
}
common_types = _common_types_default = OrderedDict([
('.rtf' , 'application/rtf'),
('.midi', 'audio/midi'),
('.mid' , 'audio/midi'),
('.jpg' , 'image/jpg'),
('.pict', 'image/pict'),
('.pct' , 'image/pict'),
('.pic' , 'image/pict'),
('.xul' , 'text/xul'),
])


_default_mime_types()
Expand Down
Loading
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy