<?xml version="1.0"?>
<ownershipDocument>

    <schemaVersion>X0306</schemaVersion>

    <documentType>4</documentType>

    <periodOfReport>2017-04-18</periodOfReport>

    <notSubjectToSection16>0</notSubjectToSection16>

    <issuer>
        <issuerCik>0001000298</issuerCik>
        <issuerName>IMPAC MORTGAGE HOLDINGS INC</issuerName>
        <issuerTradingSymbol>IMH</issuerTradingSymbol>
    </issuer>

    <reportingOwner>
        <reportingOwnerId>
            <rptOwnerCik>0000905980</rptOwnerCik>
            <rptOwnerName>PICKUP RICHARD H/</rptOwnerName>
        </reportingOwnerId>
        <reportingOwnerAddress>
            <rptOwnerStreet1>2532 DUPONT DRIVE</rptOwnerStreet1>
            <rptOwnerStreet2></rptOwnerStreet2>
            <rptOwnerCity>IRVINE</rptOwnerCity>
            <rptOwnerState>CA</rptOwnerState>
            <rptOwnerZipCode>92612</rptOwnerZipCode>
            <rptOwnerStateDescription></rptOwnerStateDescription>
        </reportingOwnerAddress>
        <reportingOwnerRelationship>
            <isDirector>0</isDirector>
            <isOfficer>0</isOfficer>
            <isTenPercentOwner>1</isTenPercentOwner>
            <isOther>0</isOther>
        </reportingOwnerRelationship>
    </reportingOwner>

    <nonDerivativeTable>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2017-04-18</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>P</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionTimeliness>
                <value></value>
            </transactionTimeliness>
            <transactionAmounts>
                <transactionShares>
                    <value>493681</value>
                </transactionShares>
                <transactionPricePerShare>
                    <value>12.66</value>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>A</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>2354146</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>See footnote.</value>
                    <footnoteId id="F1"/>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2017-04-18</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>P</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionTimeliness>
                <value></value>
            </transactionTimeliness>
            <transactionAmounts>
                <transactionShares>
                    <value>394945</value>
                </transactionShares>
                <transactionPricePerShare>
                    <value>12.66</value>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>A</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>894945</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>See footnote.</value>
                    <footnoteId id="F2"/>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2017-04-18</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>P</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionTimeliness>
                <value></value>
            </transactionTimeliness>
            <transactionAmounts>
                <transactionShares>
                    <value>691153</value>
                </transactionShares>
                <transactionPricePerShare>
                    <value>12.66</value>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>A</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>1191153</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>See footnote.</value>
                    <footnoteId id="F3"/>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
        <nonDerivativeHolding>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>100000</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>D</value>
                </directOrIndirectOwnership>
            </ownershipNature>
        </nonDerivativeHolding>
    </nonDerivativeTable>

    <derivativeTable>
        <derivativeHolding>
            <securityTitle>
                <value>Convertible Promissory Note Due 2020</value>
            </securityTitle>
            <conversionOrExercisePrice>
                <value>21.50</value>
                <footnoteId id="F4"/>
            </conversionOrExercisePrice>
            <exerciseDate>
                <value>2016-01-02</value>
                <footnoteId id="F4"/>
            </exerciseDate>
            <expirationDate>
                <value>2020-05-09</value>
                <footnoteId id="F4"/>
            </expirationDate>
            <underlyingSecurity>
                <underlyingSecurityTitle>
                    <value>Common Stock</value>
                </underlyingSecurityTitle>
                <underlyingSecurityShares>
                    <value>639535</value>
                    <footnoteId id="F4"/>
                </underlyingSecurityShares>
            </underlyingSecurity>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>13750000</value>
                    <footnoteId id="F4"/>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>See footnotes</value>
                    <footnoteId id="F4"/>
                    <footnoteId id="F5"/>
                </natureOfOwnership>
            </ownershipNature>
        </derivativeHolding>
    </derivativeTable>

    <footnotes>
        <footnote id="F1">The shares of common stock were purchased by RHP Trust, dated May 31, 2011 (the &quot;Trust&quot;), over which shares Reporting Person exercises sole voting and investment power, and the number of shares reflected in column 5 represents the aggregate number of shares of common stock owned directly by the Trust upon execution of this purchase transaction.</footnote>
        <footnote id="F2">The shares of common stock were purchased by Dito Devcar LP, over which shares Reporting Person exercises sole voting and investment power, and the number of shares reflected in column 5 represents the aggregate number of shares of common stock owned directly by Dito Devcar LP upon execution of this purchase transaction.</footnote>
        <footnote id="F3">The shares of common stock were purchased by Dito Caree LP, over which shares Reporting Person exercises sole voting and investment power, and the number of shares reflected in column 5 represents the aggregate number of shares of common stock owned directly by Dito Caree LP upon execution of this purchase transaction.</footnote>
        <footnote id="F4">As previously reported on a Form 4 filed by Reporting Person on May 11, 2015, on May 8, 2015, the Trust purchased a Convertible Promissory Note Due 2020 in the original principal amount of $13,750,000 that is convertible by the Trust at any time after January 1, 2016, and, upon conversion of the original principal amount prior to maturity at the initial conversion price of $21.50 per share (subject to adjustment in the event of stock splits, stock dividends and reclassifications), the Trust will receive 639,535 shares of common stock (subject to adjustment in the event of stock splits, stock dividends and reclassifications). The Convertible Promissory Note Due 2020 is due and payable, to the extent not converted, on or before May 9, 2020.</footnote>
        <footnote id="F5">As of April 18, 2017, Reporting Person may be deemed to beneficially own an aggregate of 5,179,779 shares of the common stock, consisting of (a) 100,000 shares owned directly, and (b) an aggregate of 5,079,779 shares owned indirectly, consisting of (i) 2,354,146 shares owned directly by the Trust, (ii) 639,535 shares that the Trust may acquire at any time after January 1, 2016 upon conversion (at the initial conversion price of $21.50 per share) of the outstanding principal balance of a Convertible Promissory Note Due 2020 owned directly by the Trust, (iii) 894,945 shares owned directly by Dito Devcar LP (over which shares Reporting Person exercises sole voting and investment power), and (iv) 1,191,153 shares owned directly by Dito Caree LP (over which shares Reporting Person exercises sole voting and investment power).</footnote>
    </footnotes>

    <remarks></remarks>

    <ownerSignature>
        <signatureName>/s/ Richard H. Pickup</signatureName>
        <signatureDate>2017-04-20</signatureDate>
    </ownerSignature>
</ownershipDocument>
