<?xml version="1.0"?>
<ownershipDocument>

    <schemaVersion>X0306</schemaVersion>

    <documentType>4</documentType>

    <periodOfReport>2015-06-30</periodOfReport>

    <notSubjectToSection16>1</notSubjectToSection16>

    <issuer>
        <issuerCik>0000722256</issuerCik>
        <issuerName>HUDSON VALLEY HOLDING CORP</issuerName>
        <issuerTradingSymbol>HVB</issuerTradingSymbol>
    </issuer>

    <reportingOwner>
        <reportingOwnerId>
            <rptOwnerCik>0000927943</rptOwnerCik>
            <rptOwnerName>LINDENBAUM MATTHEW A</rptOwnerName>
        </reportingOwnerId>
        <reportingOwnerAddress>
            <rptOwnerStreet1>C/O BASSWOOD CAPITAL MANAGEMENT L.L.C.</rptOwnerStreet1>
            <rptOwnerStreet2>645 MADISON AVENUE 10TH FLOOR</rptOwnerStreet2>
            <rptOwnerCity>NEW YORK</rptOwnerCity>
            <rptOwnerState>NY</rptOwnerState>
            <rptOwnerZipCode>10022</rptOwnerZipCode>
            <rptOwnerStateDescription></rptOwnerStateDescription>
        </reportingOwnerAddress>
        <reportingOwnerRelationship>
            <isDirector>1</isDirector>
            <isOfficer>0</isOfficer>
            <isTenPercentOwner>0</isTenPercentOwner>
            <isOther>0</isOther>
            <officerTitle></officerTitle>
            <otherText></otherText>
        </reportingOwnerRelationship>
    </reportingOwner>

    <nonDerivativeTable>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock, par value $0.20 per share (&quot;Common Stock&quot;)</value>
            </securityTitle>
            <transactionDate>
                <value>2015-06-30</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>D</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>1000</value>
                    <footnoteId id="F1"/>
                </transactionShares>
                <transactionPricePerShare>
                    <footnoteId id="F2"/>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>0</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>D</value>
                </directOrIndirectOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2015-06-30</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>D</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>152774</value>
                </transactionShares>
                <transactionPricePerShare>
                    <footnoteId id="F3"/>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>0</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                    <footnoteId id="F4"/>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>By Basswood Opportunity Partners, LP</value>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2015-06-30</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>D</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>81181</value>
                </transactionShares>
                <transactionPricePerShare>
                    <footnoteId id="F5"/>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>0</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                    <footnoteId id="F4"/>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>By Basswood Financial Fund, LP</value>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2015-06-30</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>D</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>107529</value>
                </transactionShares>
                <transactionPricePerShare>
                    <footnoteId id="F6"/>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>0</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                    <footnoteId id="F4"/>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>By Basswood Financial Fund, Inc.</value>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2015-06-30</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>S</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>40618</value>
                </transactionShares>
                <transactionPricePerShare>
                    <value>28.1029</value>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>14037</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                    <footnoteId id="F4"/>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>By Basswood Financial Long Only Fund, LP</value>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2015-06-30</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>D</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>14037</value>
                </transactionShares>
                <transactionPricePerShare>
                    <footnoteId id="F7"/>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>0</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                    <footnoteId id="F4"/>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>By Basswood Financial Long Only Fund, LP</value>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2015-06-30</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>D</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>739423</value>
                </transactionShares>
                <transactionPricePerShare>
                    <footnoteId id="F8"/>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>0</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                    <footnoteId id="F4"/>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>By Basswood Enhanced Long Short Fund, LP</value>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2015-06-30</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>D</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>90629</value>
                </transactionShares>
                <transactionPricePerShare>
                    <footnoteId id="F9"/>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>0</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                    <footnoteId id="F4"/>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>By Basswood Opportunity Fund, Inc.</value>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2015-06-30</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>D</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>730756</value>
                </transactionShares>
                <transactionPricePerShare>
                    <footnoteId id="F10"/>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>0</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                    <footnoteId id="F4"/>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>By Main Street Master, Ltd.</value>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2015-06-30</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>D</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>13127</value>
                </transactionShares>
                <transactionPricePerShare>
                    <footnoteId id="F11"/>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>0</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                    <footnoteId id="F4"/>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>By separately managed account</value>
                </natureOfOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
    </nonDerivativeTable>

    <derivativeTable></derivativeTable>

    <footnotes>
        <footnote id="F1">Includes 1,000 shares of Restricted Stock that vests 100% on February 6, 2016.  The vesting of these shares was accelerated on June 30, 2015 in connection with the consummation of the merger of the Issuer into Sterling Bancorp.</footnote>
        <footnote id="F2">Disposed of pursuant to the Agreement and Plan of Merger between the Issuer and Sterling Bancorp (the &quot;Merger Agreement&quot;) in exchange for 1,920 shares of Sterling Bancorp common stock having a market value of $14.714 per share on the effective date of the merger (with such market value calculated as the average of the closing sales price of Sterling Bancorp common stock on the New York Stock Exchange for the five trading days ending on the day preceding the closing date of the merger (the &quot;Market Value of Sterling Common Stock&quot;)).</footnote>
        <footnote id="F3">Disposed of pursuant to the Merger Agreement in exchange for 293,326 shares of Sterling Bancorp common stock having a market value of $14.714 per share on the effective date of the merger (with such market value calculated as the Market Value of Sterling Common Stock) and cash of $1.18 in lieu of a fractional share of Sterling Bancorp common stock.</footnote>
        <footnote id="F4">Basswood Opportunity Partners, LP, Basswood Financial Fund, LP, Basswood Enhanced Long Short Fund, LP, Basswood Financial Long Only Fund, LP, each a Delaware limited partnership, Basswood Opportunity Fund, Inc.,  Basswood Financial Fund, Inc., Main Street Master, Ltd., each a Cayman Islands corporation (collectively, the &quot;Funds&quot;) directly own shares of Common Stock.  Basswood Capital Management, L.L.C., a Delaware limited liability company (the &quot;Management Company&quot;) is the investment manager or adviser to the Funds and certain separately managed accounts (the &quot;SMAs&quot;).  Matthew A. Lindenbaum is managing member of the Management Company, and along with the other managing member, controls the business activities of the Management Company.  Mr. Lindenbaum disclaims beneficial ownership of all shares of the Common Stock held directly by the Funds and the SMAs except to the extent of any indirect pecuniary interest therein.</footnote>
        <footnote id="F5">Disposed of pursuant to the Merger Agreement in exchange for 155,867 shares of Sterling Bancorp common stock having a market value of $14.714 per share on the effective date of the merger (with such market value calculated as the Market Value of Sterling Common Stock) and cash of $7.65 in lieu of a fractional share of Sterling Bancorp common stock.</footnote>
        <footnote id="F6">Disposed of pursuant to the Merger Agreement in exchange for 206,455 shares of Sterling Bancorp common stock having a market value of $14.714 per share on the effective date of the merger (with such market value calculated as the Market Value of Sterling Common Stock) and cash of $10.01 in lieu of a fractional share of Sterling Bancorp common stock.</footnote>
        <footnote id="F7">Disposed of pursuant to the Merger Agreement in exchange for 26,951 shares of Sterling Bancorp common stock having a market value of $14.714 per share on the effective date of the merger (with such market value calculated as the Market Value of Sterling Common Stock) and cash of $0.59 in lieu of a fractional share of Sterling Bancorp common stock.</footnote>
        <footnote id="F8">Disposed of pursuant to the Merger Agreement in exchange for 1,419,692 shares of Sterling Bancorp common stock having a market value of $14.714 per share on the effective date of the merger (with such market value calculated as the Market Value of Sterling Common Stock) and cash of $2.35 in lieu of a fractional share of Sterling Bancorp common stock.</footnote>
        <footnote id="F9">Disposed of pursuant to the Merger Agreement in exchange for 174,007 shares of Sterling Bancorp common stock having a market value of $14.714 per share on the effective date of the merger (with such market value calculated as the Market Value of Sterling Common Stock) and cash of $10.01 in lieu of a fractional share of Sterling Bancorp common stock.</footnote>
        <footnote id="F10">Disposed of pursuant to the Merger Agreement in exchange for 1,403,051 shares of Sterling Bancorp common stock having a market value of $14.714 per share on the effective date of the merger (with such market value calculated as the Market Value of Sterling Common Stock) and cash of $7.65 in lieu of a fractional share of Sterling Bancorp common stock.</footnote>
        <footnote id="F11">Disposed of pursuant to the Merger Agreement in exchange for 25,203 shares of Sterling Bancorp common stock having a market value of $14.714 per share on the effective date of the merger (with such market value calculated as the Market Value of Sterling Common Stock) and cash of $12.36 in lieu of a fractional share of Sterling Bancorp common stock.</footnote>
    </footnotes>

    <remarks></remarks>

    <ownerSignature>
        <signatureName>/s/ Matthew A. Lindenbaum</signatureName>
        <signatureDate>2015-07-02</signatureDate>
    </ownerSignature>
</ownershipDocument>
