<?xml version="1.0"?>
<ownershipDocument>

    <schemaVersion>X0304</schemaVersion>

    <documentType>4</documentType>

    <periodOfReport>2011-10-21</periodOfReport>

    <notSubjectToSection16>1</notSubjectToSection16>

    <issuer>
        <issuerCik>0000810689</issuerCik>
        <issuerName>BANK OF GRANITE CORP</issuerName>
        <issuerTradingSymbol>GRAN</issuerTradingSymbol>
    </issuer>

    <reportingOwner>
        <reportingOwnerId>
            <rptOwnerCik>0001210354</rptOwnerCik>
            <rptOwnerName>GAITHER HUGH R</rptOwnerName>
        </reportingOwnerId>
        <reportingOwnerAddress>
            <rptOwnerStreet1>603 NORTH MAIN AVENUE</rptOwnerStreet1>
            <rptOwnerStreet2></rptOwnerStreet2>
            <rptOwnerCity>NEWTON</rptOwnerCity>
            <rptOwnerState>NC</rptOwnerState>
            <rptOwnerZipCode>28658</rptOwnerZipCode>
            <rptOwnerStateDescription></rptOwnerStateDescription>
        </reportingOwnerAddress>
        <reportingOwnerRelationship>
            <isDirector>1</isDirector>
            <isOfficer>0</isOfficer>
            <isTenPercentOwner>0</isTenPercentOwner>
            <isOther>0</isOther>
            <officerTitle></officerTitle>
            <otherText></otherText>
        </reportingOwnerRelationship>
    </reportingOwner>

    <nonDerivativeTable>
        <nonDerivativeTransaction>
            <securityTitle>
                <value>Common Stock</value>
            </securityTitle>
            <transactionDate>
                <value>2011-10-21</value>
            </transactionDate>
            <deemedExecutionDate></deemedExecutionDate>
            <transactionCoding>
                <transactionFormType>4</transactionFormType>
                <transactionCode>D</transactionCode>
                <equitySwapInvolved>0</equitySwapInvolved>
            </transactionCoding>
            <transactionAmounts>
                <transactionShares>
                    <value>749</value>
                </transactionShares>
                <transactionPricePerShare>
                    <footnoteId id="F1"/>
                </transactionPricePerShare>
                <transactionAcquiredDisposedCode>
                    <value>D</value>
                </transactionAcquiredDisposedCode>
            </transactionAmounts>
            <postTransactionAmounts>
                <sharesOwnedFollowingTransaction>
                    <value>0</value>
                </sharesOwnedFollowingTransaction>
            </postTransactionAmounts>
            <ownershipNature>
                <directOrIndirectOwnership>
                    <value>D</value>
                </directOrIndirectOwnership>
            </ownershipNature>
        </nonDerivativeTransaction>
    </nonDerivativeTable>

    <derivativeTable></derivativeTable>

    <footnotes>
        <footnote id="F1">Disposed of pursuant to that certain Agreement and Plan of Merger, dated as of April 26, 2011, as amended by Amendment No. 1, dated as of June 16, 2011, and Amendment No. 2, dated as of August 15, 2011, by and among FNB United Corp., Gamma Merger Corporation and the issuer (the &quot;Merger&quot;) in exchange for 2,528 shares of FNB United Corp. common stock having a market value of $0.38 per share on the effective date of the Merger.</footnote>
    </footnotes>

    <remarks></remarks>

    <ownerSignature>
        <signatureName>/s/Kathy K. Starnes, Attorney in Fact</signatureName>
        <signatureDate>2011-10-25</signatureDate>
    </ownerSignature>
</ownershipDocument>
