<?xml version="1.0"?>
<ownershipDocument>

    <schemaVersion>X0306</schemaVersion>

    <documentType>4/A</documentType>

    <periodOfReport>2011-10-31</periodOfReport>

    <dateOfOriginalSubmission>2011-11-02</dateOfOriginalSubmission>

    <notSubjectToSection16>0</notSubjectToSection16>

    <issuer>
        <issuerCik>0000811808</issuerCik>
        <issuerName>SUMMIT FINANCIAL GROUP INC</issuerName>
        <issuerTradingSymbol>SMMF</issuerTradingSymbol>
    </issuer>

    <reportingOwner>
        <reportingOwnerId>
            <rptOwnerCik>0001486521</rptOwnerCik>
            <rptOwnerName>George Georgette R.</rptOwnerName>
        </reportingOwnerId>
        <reportingOwnerAddress>
            <rptOwnerStreet1>PO BOX 8523</rptOwnerStreet1>
            <rptOwnerStreet2></rptOwnerStreet2>
            <rptOwnerCity>CHARLESTON</rptOwnerCity>
            <rptOwnerState>WV</rptOwnerState>
            <rptOwnerZipCode>25303</rptOwnerZipCode>
            <rptOwnerStateDescription></rptOwnerStateDescription>
        </reportingOwnerAddress>
        <reportingOwnerRelationship>
            <isDirector>1</isDirector>
            <isOfficer>0</isOfficer>
            <isTenPercentOwner>0</isTenPercentOwner>
            <isOther>0</isOther>
            <officerTitle></officerTitle>
            <otherText></otherText>
        </reportingOwnerRelationship>
    </reportingOwner>

    <nonDerivativeTable></nonDerivativeTable>

    <derivativeTable>
        <derivativeTransaction>
            <securityTitle>
                <value>Subscription Rights (right to buy)</value>
            </securityTitle>
            <conversionOrExercisePrice>
                <value>500.00</value>
            </conversionOrExercisePrice>
            <transactionDate>
                <value>2011-10-31</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>X</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>118</value>
                    <footnoteId id="F2"/>
                </transactionShares>
                <transactionPricePerShare>
                    <value>500</value>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <exerciseDate>
                <value>2011-07-01</value>
            </exerciseDate>
            <expirationDate>
                <value>2011-10-31</value>
            </expirationDate>
            <underlyingSecurity>
                <underlyingSecurityTitle>
                    <value>Preferred Stock</value>
                </underlyingSecurityTitle>
                <underlyingSecurityShares>
                    <value>118</value>
                    <footnoteId id="F2"/>
                </underlyingSecurityShares>
            </underlyingSecurity>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>118</value>
                    <footnoteId id="F2"/>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>By 401(k) Plan FBO Spouse</value>
                </natureOfOwnership>
            </ownershipNature>
        </derivativeTransaction>
        <derivativeTransaction>
            <securityTitle>
                <value>8% Non-Cumulative Convertible Preferred Stock, Series 2011</value>
            </securityTitle>
            <conversionOrExercisePrice>
                <value>4.00</value>
            </conversionOrExercisePrice>
            <transactionDate>
                <value>2011-10-31</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>P</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>118</value>
                    <footnoteId id="F2"/>
                </transactionShares>
                <transactionPricePerShare>
                    <value>500</value>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>A</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <exerciseDate>
                <value>2012-03-01</value>
                <footnoteId id="F1"/>
            </exerciseDate>
            <expirationDate>
                <value>2021-06-01</value>
            </expirationDate>
            <underlyingSecurity>
                <underlyingSecurityTitle>
                    <value>Common Stock</value>
                </underlyingSecurityTitle>
                <underlyingSecurityShares>
                    <value>14750</value>
                    <footnoteId id="F2"/>
                </underlyingSecurityShares>
            </underlyingSecurity>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>14750</value>
                    <footnoteId id="F2"/>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>I</value>
                </directOrIndirectOwnership>
                <natureOfOwnership>
                    <value>By 401(k) Plan FBO Spouse</value>
                </natureOfOwnership>
            </ownershipNature>
        </derivativeTransaction>
    </derivativeTable>

    <footnotes>
        <footnote id="F1">The 2009 Series Preferred Stock and 2011 Series Preferred Stock may be converted at the holder's option on any dividend payment date.</footnote>
        <footnote id="F2">On November 2, 2011, the reporting person filed a Form 4 reporting the acquisition of 100 shares of 2011 Preferred Stock by 401(k) Plan FBO Spouse.  The correct number of shares of Preferred Stock acquired was 118, which are convertible into 14,750 shares of Common Stock.</footnote>
    </footnotes>

    <remarks></remarks>

    <ownerSignature>
        <signatureName>Teresa D. Ely, Lmtd POA Attorney-in-Fact</signatureName>
        <signatureDate>2015-07-23</signatureDate>
    </ownerSignature>
</ownershipDocument>
