[pull] master from sqlmapproject:master

data/xml/queries.xml

@@ -83,7 +83,7 @@
     </dbms>
 
     <dbms value="PostgreSQL">
-        <cast query="CAST(%s AS VARCHAR(10000))"/>
+        <cast query="CAST(%s AS TEXT)"/>
         <length query="LENGTH(%s)"/>
         <!-- NOTE: PostgreSQL does not like COALESCE with different data-types (e.g. COALESCE(id,' ')) -->
         <isnull query="COALESCE(%s::text,' ')"/>
@@ -119,8 +119,8 @@
             <blind query="SELECT DISTINCT(passwd) FROM pg_shadow WHERE usename='%s' OFFSET %d LIMIT 1" count="SELECT COUNT(DISTINCT(passwd)) FROM pg_shadow WHERE usename='%s'"/>
         </passwords>
         <privileges>
-            <inband query="SELECT usename,(CASE WHEN usecreatedb THEN 1 ELSE 0 END),(CASE WHEN usesuper THEN 1 ELSE 0 END),(CASE WHEN usecatupd THEN 1 ELSE 0 END) FROM pg_user" condition="usename"/>
-            <blind query="SELECT (CASE WHEN usecreatedb THEN 1 ELSE 0 END),(CASE WHEN usesuper THEN 1 ELSE 0 END),(CASE WHEN usecatupd THEN 1 ELSE 0 END) FROM pg_user WHERE usename='%s' OFFSET %d LIMIT 1" count="SELECT COUNT(DISTINCT(usename)) FROM pg_user WHERE usename='%s'"/>
+            <inband query="SELECT usename,(CASE WHEN usecreatedb THEN 1 ELSE 0 END),(CASE WHEN usesuper THEN 1 ELSE 0 END),(CASE WHEN userepl THEN 1 ELSE 0 END) FROM pg_user" condition="usename"/>
+            <blind query="SELECT (CASE WHEN usecreatedb THEN 1 ELSE 0 END),(CASE WHEN usesuper THEN 1 ELSE 0 END),(CASE WHEN userepl THEN 1 ELSE 0 END) FROM pg_user WHERE usename='%s' OFFSET %d LIMIT 1" count="SELECT COUNT(DISTINCT(usename)) FROM pg_user WHERE usename='%s'"/>
         </privileges>
         <roles/>
         <statements>
@@ -163,7 +163,7 @@
     </dbms>
 
     <dbms value="Microsoft SQL Server">
-        <cast query="CAST(%s AS NVARCHAR(4000))"/>
+        <cast query="CAST(%s AS NVARCHAR(MAX))"/>
         <length query="LTRIM(STR(LEN(%s+'.')-1))"/>
         <isnull query="ISNULL(%s,' ')"/>
         <delimiter query="+"/>
@@ -222,7 +222,7 @@
         </columns>
         <dump_table>
             <inband query="SELECT %s FROM %s.%s"/>
-            <blind query="SELECT MIN(%s) FROM %s WHERE CONVERT(NVARCHAR(4000),%s)>'%s'" query2="SELECT MAX(%s) FROM %s WHERE CONVERT(NVARCHAR(4000),%s) LIKE '%s'" query3="SELECT %s FROM (SELECT %s, ROW_NUMBER() OVER (ORDER BY (SELECT 1)) AS CAP FROM %s)x WHERE CAP=%d" count="SELECT LTRIM(STR(COUNT(*))) FROM %s" count2="SELECT LTRIM(STR(COUNT(DISTINCT(%s)))) FROM %s" keyset_first="SELECT MIN(%s) FROM %s" keyset_next="SELECT MIN(%s) FROM %s WHERE %s>'%s'" keyset_by="SELECT MAX(%s) FROM %s WHERE %s='%s'" keyset_seed="SELECT %s FROM %s ORDER BY %s OFFSET %d ROWS FETCH NEXT 1 ROWS ONLY" keyset_ordered="SELECT TOP 1 %s FROM %s WHERE %s ORDER BY %s" keyset_where="SELECT MAX(%s) FROM %s WHERE %s"/>
+            <blind query="SELECT MIN(%s) FROM %s WHERE CONVERT(NVARCHAR(4000),%s)>'%s'" query2="SELECT MAX(%s) FROM %s WHERE CONVERT(NVARCHAR(4000),%s) LIKE '%s'" query3="SELECT %s FROM (SELECT %s, ROW_NUMBER() OVER (ORDER BY %s) AS CAP FROM %s)x WHERE CAP=%d" count="SELECT LTRIM(STR(COUNT(*))) FROM %s" count2="SELECT LTRIM(STR(COUNT(DISTINCT(%s)))) FROM %s" keyset_first="SELECT MIN(%s) FROM %s" keyset_next="SELECT MIN(%s) FROM %s WHERE %s>'%s'" keyset_by="SELECT MAX(%s) FROM %s WHERE %s='%s'" keyset_seed="SELECT %s FROM %s ORDER BY %s OFFSET %d ROWS FETCH NEXT 1 ROWS ONLY" keyset_ordered="SELECT TOP 1 %s FROM %s WHERE %s ORDER BY %s" keyset_where="SELECT MAX(%s) FROM %s WHERE %s"/>
             <primary_key count="SELECT COUNT(*) FROM sys.indexes i JOIN sys.index_columns ic ON i.object_id=ic.object_id AND i.index_id=ic.index_id WHERE i.is_primary_key=1 AND i.object_id=OBJECT_ID('%s.dbo.%s')" query="SELECT name FROM (SELECT c.name AS name, ROW_NUMBER() OVER (ORDER BY ic.key_ordinal) AS rn FROM sys.indexes i JOIN sys.index_columns ic ON i.object_id=ic.object_id AND i.index_id=ic.index_id JOIN sys.columns c ON ic.object_id=c.object_id AND c.column_id=ic.column_id WHERE i.is_primary_key=1 AND i.object_id=OBJECT_ID('%s.dbo.%s')) x WHERE rn=%d+1"/>
         </dump_table>
         <search_db>
@@ -384,7 +384,7 @@
         </columns>
         <dump_table>
             <inband query="SELECT %s FROM %s"/>
-            <blind query="SELECT %s FROM %s LIMIT %d,1" count="SELECT COUNT(*) FROM %s" rowid="rowid" keyset_first="SELECT MIN(%s) FROM %s" keyset_next="SELECT MIN(%s) FROM %s WHERE %s>'%s'" keyset_by="SELECT MAX(%s) FROM %s WHERE %s='%s'" keyset_seed="SELECT %s FROM %s ORDER BY %s LIMIT 1 OFFSET %d" keyset_ordered="SELECT %s FROM %s WHERE %s ORDER BY %s LIMIT 1" keyset_where="SELECT MAX(%s) FROM %s WHERE %s"/>
+            <blind query="SELECT %s FROM %s ORDER BY rowid LIMIT %d,1" count="SELECT COUNT(*) FROM %s" rowid="rowid" keyset_first="SELECT MIN(%s) FROM %s" keyset_next="SELECT MIN(%s) FROM %s WHERE %s>'%s'" keyset_by="SELECT MAX(%s) FROM %s WHERE %s='%s'" keyset_seed="SELECT %s FROM %s ORDER BY %s LIMIT 1 OFFSET %d" keyset_ordered="SELECT %s FROM %s WHERE %s ORDER BY %s LIMIT 1" keyset_where="SELECT MAX(%s) FROM %s WHERE %s"/>
         </dump_table>
         <search_db/>
         <search_table>
@@ -959,8 +959,8 @@
 
     <dbms value="Apache Derby">
         <!-- NOTE: CHAR(%s) causes 'A truncation error was encountered trying to shrink CHAR' -->
-        <cast query="RTRIM(CAST(%s AS CHAR(254)))"/>
-        <length query="LENGTH(RTRIM(CAST(%s AS CHAR(254))))"/>
+        <cast query="CAST(%s AS VARCHAR(32672))"/>
+        <length query="LENGTH(CAST(%s AS VARCHAR(32672)))"/>
         <isnull query="COALESCE(%s,' ')"/>
         <delimiter query="||"/>
         <limit query="OFFSET %d ROWS FETCH FIRST %d ROWS ONLY"/>
@@ -1158,7 +1158,7 @@
         <substring query="SUBSTR(%s,%d,%d)"/>
         <concatenate query="%s||%s"/>
         <case query="SELECT (CASE WHEN (%s) THEN '1' ELSE '0' END)"/>
-        <hex query="TO_HEX(%s)"/>
+        <hex query="TO_HEX(CAST(%s AS VARBINARY))"/>
         <inference query="CODEPOINT(SUBSTR((%s),%d,1))>%d" dbms_version="&gt;=0.178" query2="SUBSTR((%s),%d,1)>'%c'"/>/>
         <banner query="version()"/>
         <current_user query="CURRENT_USER"/>

lib/core/agent.py

@@ -508,10 +508,26 @@ def nullAndCastField(self, field):
         if field and Backend.getIdentifiedDbms():
             rootQuery = queries[Backend.getIdentifiedDbms()]
 
+            kb.binaryField = conf.binaryFields and field in conf.binaryFields
+            hexConvert = conf.hexConvert or kb.binaryField
+
+            # For BINARY fields, wrap the RAW column with the hex function rather than the text-casted one:
+            # text-casting binary first (e.g. CAST(<binary> AS NCHAR) on MySQL) NULLs non-text bytes, so HEX()
+            # would encode the NULL placeholder and silently lose the value (e.g. binary-stored password
+            # hashes). This is limited to binary fields: the blanket '--hex' keeps the cast-first path because
+            # raw-hexing a numeric/temporal column uses the DBMS's numeric HEX semantics (MySQL HEX(255)='FF'),
+            # which is NOT the hex of its string representation. Every DBMS hex function takes binary directly
+            # EXCEPT PostgreSQL's CONVERT_TO() (needs text), so PostgreSQL stays on the cast-first path too.
+            hexRaw = kb.binaryField and not Backend.isDbms(DBMS.PGSQL)
+
             if field.startswith("(CASE") or field.startswith("(IIF") or conf.noCast and not (field.startswith("COUNT(") and Backend.getIdentifiedDbms() == DBMS.MSSQL):
                 nulledCastedField = field
+                if hexConvert:
+                    nulledCastedField = self.hexConvertField(nulledCastedField)
             else:
-                if not (Backend.isDbms(DBMS.SQLITE) and not isDBMSVersionAtLeast('3')):
+                if hexRaw:
+                    nulledCastedField = self.hexConvertField(field)
+                elif not (Backend.isDbms(DBMS.SQLITE) and not isDBMSVersionAtLeast('3')):
                     nulledCastedField = rootQuery.cast.query % field
 
                 if re.search(r"COUNT\(", field) and Backend.getIdentifiedDbms() in (DBMS.RAIMA,):
@@ -521,9 +537,8 @@ def nullAndCastField(self, field):
                 else:
                     nulledCastedField = rootQuery.isnull.query % nulledCastedField
 
-            kb.binaryField = conf.binaryFields and field in conf.binaryFields
-            if conf.hexConvert or kb.binaryField:
-                nulledCastedField = self.hexConvertField(nulledCastedField)
+                if hexConvert and not hexRaw:
+                    nulledCastedField = self.hexConvertField(nulledCastedField)
 
         if suffix:
             nulledCastedField += suffix

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.7.73"
+VERSION = "1.10.7.77"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
@@ -879,15 +879,17 @@
 # Warn user of possible delay due to large page dump in full UNION query injections
 LARGE_OUTPUT_THRESHOLD = 1024 ** 2
 
-# On huge tables there is a considerable slowdown if every row retrieval requires ORDER BY (most noticable in table dumping using ERROR injections)
-SLOW_ORDER_COUNT_THRESHOLD = 10000
-
 # Give up on hash recognition if nothing was found in first given number of rows
 HASH_RECOGNITION_QUIT_THRESHOLD = 1000
 
 # Regular expression used for automatic hex conversion and hash cracking of (RAW) binary column values
 HASH_BINARY_COLUMNS_REGEX = r"(?i)pass|psw|hash"
 
+# Regular expression matching (declared) binary column types, used to auto-hex their values during dumping
+# so raw bytes (e.g. password hashes stored in binary form) are not silently truncated at NUL / mangled by
+# the text extraction channel (mirrors a manual '--binary-fields', using the already-fetched column type)
+BINARY_FIELDS_TYPE_REGEX = r"(?i)binary|blob|bytea|image|\braw\b"
+
 # Maximum number of redirections to any single URL - this is needed because of the state that cookies introduce
 MAX_SINGLE_URL_REDIRECTIONS = 4
 

lib/core/target.py

@@ -214,6 +214,22 @@ def process(match, repl):
                     conf.data = conf.data.replace(kb.customInjectionMark, ASTERISK_MARKER)
                     conf.data = re.sub(r"(<(?P<name>[^>]+)( [^<]*)?>)([^<]+)(</\2)", functools.partial(process, repl=r"\g<1>\g<4>%s\g<5>" % kb.customInjectionMark), conf.data)
 
+                    # Also expose XML attribute values (e.g. <item id="1">) as injection points, not just
+                    # element text (Reference: https://github.com/sqlmapproject/sqlmap/issues/5993). Done per
+                    # opening tag (skipping <?...?> declarations, <!...> and closing tags) so every attribute
+                    # is covered while the XML declaration/namespaces are left intact.
+                    def processAttributes(match):
+                        def _attr(m):
+                            name = m.group("name")
+                            if name == "xmlns" or name.startswith("xmlns:"):  # namespace declarations are not injection points
+                                return m.group(0)
+                            if conf.testParameter and name not in (removePostHintPrefix(_) for _ in conf.testParameter):
+                                return m.group(0)
+                            hintNames.append(("%s%s%s" % (m.group(1), m.group(3), m.group(4)), name))
+                            return "%s%s%s%s%s" % (m.group(1), m.group(3), m.group(4), kb.customInjectionMark, m.group(5))
+                        return re.sub(r'(\s(?P<name>[\w:.-]+)\s*=\s*)(["\'])([^"\'<>]*)(["\'])', _attr, match.group(0))
+                    conf.data = re.sub(r"(?s)<[^>?!/][^>]*>", processAttributes, conf.data)
+
         elif re.search(MULTIPART_RECOGNITION_REGEX, conf.data):
             message = "Multipart-like data found in %s body. " % conf.method
             message += "Do you want to process it? [Y/n/q] "

lib/techniques/error/use.py

@@ -29,7 +29,6 @@
 from lib.core.common import isListLike
 from lib.core.common import isNumPosStrValue
 from lib.core.common import listToStrValue
-from lib.core.common import readInput
 from lib.core.common import unArrayizeValue
 from lib.core.common import wasLastResponseHTTPError
 from lib.core.compat import xrange
@@ -51,7 +50,6 @@
 from lib.core.settings import NULL
 from lib.core.settings import PARTIAL_VALUE_MARKER
 from lib.core.settings import ROTATING_CHARS
-from lib.core.settings import SLOW_ORDER_COUNT_THRESHOLD
 from lib.core.settings import SQL_SCALAR_REGEX
 from lib.core.settings import TURN_OFF_RESUME_INFO_LIMIT
 from lib.core.threads import getCurrentThreadData
@@ -322,7 +320,7 @@ def errorUse(expression, dump=False):
     # entry at a time
     # NOTE: we assume that only queries that get data from a table can
     # return multiple entries
-    if (dump and (conf.limitStart or conf.limitStop)) or (" FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and ("(CASE" not in expression.upper() or ("(CASE" in expression.upper() and "WHEN use" in expression))) and not re.search(SQL_SCALAR_REGEX, expression, re.I):
+    if not re.search(SQL_SCALAR_REGEX, expression, re.I) and ((dump and (conf.limitStart or conf.limitStop)) or (" FROM " in expression.upper() and ((Backend.getIdentifiedDbms() not in FROM_DUMMY_TABLE) or (Backend.getIdentifiedDbms() in FROM_DUMMY_TABLE and not expression.upper().endswith(FROM_DUMMY_TABLE[Backend.getIdentifiedDbms()]))) and ("(CASE" not in expression.upper() or ("(CASE" in expression.upper() and "WHEN use" in expression)))):
         expression, limitCond, topLimit, startLimit, stopLimit = agent.limitCondition(expression, dump)
 
         if limitCond:
@@ -367,13 +365,10 @@ def errorUse(expression, dump=False):
                 return value
 
             if isNumPosStrValue(count) and int(count) > 1:
-                if " ORDER BY " in expression and (stopLimit - startLimit) > SLOW_ORDER_COUNT_THRESHOLD:
-                    message = "due to huge table size do you want to remove "
-                    message += "ORDER BY clause gaining speed over consistency? [y/N] "
-
-                    if readInput(message, default='N', boolean=True):
-                        expression = expression[:expression.index(" ORDER BY ")]
-
+                # NOTE: the ORDER BY clause is deliberately NOT stripped here. This path fetches each column
+                # of a row in a separate LIMIT/OFFSET query, so dropping ORDER BY would let the per-column
+                # offsets resolve to different physical rows and silently misalign cells across the row. Huge
+                # tables are handled cheaply and safely by keyset (seek) pagination (see plugins/generic/entries.py).
                 numThreads = min(conf.threads, (stopLimit - startLimit))
 
                 threadData = getCurrentThreadData()

lib/techniques/union/test.py

@@ -61,14 +61,37 @@ def _findUnionCharCount(comment, place, parameter, value, prefix, suffix, where=
 
     @stackedmethod
     def _orderByTechnique(lowerCount=None, upperCount=None):
-        def _orderByTest(cols):
+        def _orderByProbe(cols):
             query = agent.prefixQuery("ORDER BY %d" % cols, prefix=prefix)
             query = agent.suffixQuery(query, suffix=suffix, comment=comment)
             payload = agent.payload(newValue=query, place=place, parameter=parameter, where=where)
             page, headers, code = Request.queryPage(payload, place=place, content=True, raise404=False)
-            return not any(re.search(_, page or "", re.I) and not re.search(_, kb.pageTemplate or "", re.I) for _ in ("(warning|error):", "order (by|clause)", "unknown column", "failed")) and not kb.heavilyDynamic and comparison(page, headers, code) or re.search(r"data types cannot be compared or sorted", page or "", re.I) is not None
+            errored = any(re.search(_, page or "", re.I) and not re.search(_, kb.pageTemplate or "", re.I) for _ in ("(warning|error):", "order (by|clause)", "unknown column", "failed"))
+            return page, headers, code, errored
+
+        lowCount = 1 if lowerCount is None else lowerCount
+        highCount = randomInt() if upperCount is None else upperCount + 1
+
+        lowPage, lowHeaders, lowCode, lowErrored = _orderByProbe(lowCount)
+        highPage, highHeaders, highCode, highErrored = _orderByProbe(highCount)
+
+        # When an out-of-range ORDER BY position visibly errors on this target, the column count can be
+        # bisected on error presence alone - immune to result-set shifts caused by the injection context
+        # (e.g. a LIKE '%...%' search whose matches change once the string literal is closed), which would
+        # otherwise make the response-similarity oracle reject a still-valid position.
+        errorOracle = highErrored and not lowErrored
+
+        def _orderByValid(page, headers, code, errored):
+            if re.search(r"data types cannot be compared or sorted", page or "", re.I):
+                return True
+            if errored:
+                return False
+            return errorOracle or (not kb.heavilyDynamic and comparison(page, headers, code))
+
+        def _orderByTest(cols):
+            return _orderByValid(*_orderByProbe(cols))
 
-        if _orderByTest(1 if lowerCount is None else lowerCount) and not _orderByTest(randomInt() if upperCount is None else upperCount + 1):
+        if _orderByValid(lowPage, lowHeaders, lowCode, lowErrored) and not _orderByValid(highPage, highHeaders, highCode, highErrored):
             infoMsg = "'ORDER BY' technique appears to be usable. "
             infoMsg += "This should reduce the time needed "
             infoMsg += "to find the right number "

lib/utils/keysetdump.py

@@ -189,12 +189,12 @@ def _dumpSingle(tbl, colList, count, cursor, tableRef, entries, lengths):
     produced = 0
 
     while produced < target:
-        if pivotValue is None:
-            query = blind.keyset_first % (field, tableRef)
-        else:
-            query = _embed(blind.keyset_next, pivotValue, field, tableRef, field)
-
-        query = agent.whereQuery(query)
+        # Advance with ORDER BY ... LIMIT 1 (like the composite path), NOT MIN(): the value-extraction
+        # casts the aggregated column to VARCHAR *inside* MIN(), yielding a LEXICAL minimum ('10' after
+        # '1') that disagrees with the numeric '>' comparison and silently skips rows (2..9, 11..). The
+        # ORDER BY is on the raw (numeric) column, so the next cursor value is the true successor.
+        condition = "1=1" if pivotValue is None else "%s>%s" % (field, _lit(pivotValue))
+        query = agent.whereQuery(blind.keyset_ordered % (field, tableRef, condition, field))
         value = unArrayizeValue(inject.getValue(query))
 
         if isNoneValue(value) or value == NULL: